From 10338391b8a624a4df65a6d7d3a3b40b01324550 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 12 Jun 2026 23:11:00 +0800 Subject: [PATCH 1/8] feat(core): migrate Computer Use to cua-driver (cross-platform) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the open-computer-use (ocu) npm backend with cua-driver-rs, the Rust driver from trycua/cua — a background, no-focus-stealing native automation driver speaking MCP over stdio. Computer Use is experimental; this is a clean replace with no backend flag. Distribution - Download the pinned, signed + notarized binary into ~/.qwen/computer-use/ (OSS mirror primary, GitHub release fallback, QWEN_COMPUTER_USE_DOWNLOAD_HOST override), sha256-verified against the release checksums.txt before extraction. Per-platform install - macOS: tar.gz carrying CuaDriver.app; spawn the in-bundle binary so cua-driver's TCC auto-relaunch attributes Accessibility / Screen Recording to com.trycua.driver instead of the launching terminal. The first-use flow grants ONE permission at a time via a no-gate status daemon polled every 5s. - Linux: bare-binary tarball (lone cua-driver at the archive root). - Windows: .zip extracted via OS tools (bsdtar, then PowerShell Expand-Archive) — no new dependency. Runtime - Expose the full 35-tool cua-driver surface (no curation). - Transparent reconnect + retry on daemon restart / transport close (the Screen-Recording-grant restart that broke first use). CI - Add a three-OS (windows/ubuntu/macos) download+extract smoke job — the only real coverage of the Linux/Windows download paths, which can't run on the macOS dev box. --- .github/workflows/ci.yml | 36 + .../src/tools/computer-use/bootstrap.test.ts | 260 ++--- .../core/src/tools/computer-use/bootstrap.ts | 469 +++++---- .../src/tools/computer-use/client.test.ts | 43 +- .../core/src/tools/computer-use/client.ts | 139 +-- .../src/tools/computer-use/constants.test.ts | 173 ++- .../core/src/tools/computer-use/constants.ts | 222 +++- .../computer-use/downloader.smoke.test.ts | 50 + .../src/tools/computer-use/downloader.test.ts | 161 +++ .../core/src/tools/computer-use/downloader.ts | 313 ++++++ .../computer-use/permission-detector.test.ts | 50 +- .../tools/computer-use/permission-detector.ts | 37 +- .../tools/computer-use/registration.test.ts | 12 +- .../src/tools/computer-use/schemas.test.ts | 73 +- .../core/src/tools/computer-use/schemas.ts | 991 ++++++++++++++++-- .../core/src/tools/computer-use/tool.test.ts | 139 ++- packages/core/src/tools/computer-use/tool.ts | 51 +- 17 files changed, 2471 insertions(+), 748 deletions(-) create mode 100644 packages/core/src/tools/computer-use/downloader.smoke.test.ts create mode 100644 packages/core/src/tools/computer-use/downloader.test.ts create mode 100644 packages/core/src/tools/computer-use/downloader.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44c3dc3fb7b..19ca29e06bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,6 +228,42 @@ jobs: name: 'coverage-reports-${{ matrix.node-version }}-${{ matrix.os }}' path: 'packages/*/coverage' + computer_use_download_smoke: + name: 'Computer Use download smoke (${{ matrix.os }})' + needs: 'classify_pr' + if: "${{ !cancelled() && needs.classify_pr.outputs.skip_ci != 'true' }}" + runs-on: '${{ matrix.os }}' + permissions: + contents: 'read' + strategy: + fail-fast: false # See every OS's download result, not just the first failure + matrix: + os: + - 'windows-latest' + - 'ubuntu-latest' + - 'macos-latest' + steps: + - name: 'Checkout' + uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2 + + - name: 'Set up Node.js' + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + with: + node-version: '22.x' + cache: 'npm' + cache-dependency-path: 'package-lock.json' + + - name: 'Install dependencies' + run: 'npm ci --prefer-offline --no-audit --progress=false' + + # Real download + checksum verify + OS extraction (bsdtar/PowerShell on + # Windows, tar on Linux/macOS). This is the only coverage of the Linux + # bare-binary and Windows .zip paths, which can't run on the mac dev box. + - name: 'Download + extract the pinned cua-driver' + env: + CUA_DOWNLOAD_SMOKE: '1' + run: 'npx vitest run packages/core/src/tools/computer-use/downloader.smoke.test.ts' + post_coverage_comment: name: 'Post Coverage Comment' runs-on: 'ubuntu-latest' diff --git a/packages/core/src/tools/computer-use/bootstrap.test.ts b/packages/core/src/tools/computer-use/bootstrap.test.ts index 4a15962ecdf..87e6705cd77 100644 --- a/packages/core/src/tools/computer-use/bootstrap.test.ts +++ b/packages/core/src/tools/computer-use/bootstrap.test.ts @@ -10,35 +10,43 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { runBootstrap, - parseDoctorStdout, + parsePermissionsStatus, type BootstrapDeps, + type StatusDaemon, } from './bootstrap.js'; -function makeFakeClient(opts: { startThrows?: Error } = {}) { - const start = vi.fn(async () => { - if (opts.startThrows) throw opts.startThrows; - }); +const KEY = 'cua-driver-rs@0.5.2'; + +function makeFakeClient() { + const start = vi.fn(async () => {}); + const stop = vi.fn(async () => {}); return { - isStarted: vi.fn(() => start.mock.calls.length > 0), + isStarted: vi.fn(() => start.mock.calls.length > stop.mock.calls.length), start, + stop, callTool: vi.fn(), - stop: vi.fn(), }; } describe('runBootstrap', () => { let tmpHome: string; + let daemon: StatusDaemon & { kill: ReturnType }; let deps: BootstrapDeps; beforeEach(() => { tmpHome = mkdtempSync(join(tmpdir(), 'qwen-cu-bs-')); + daemon = { kill: vi.fn() }; deps = { homeDir: tmpHome, - packageSpec: '@qwen-code/open-computer-use@^0.3.0', + approvalKey: KEY, platform: 'darwin', promptInstallApproval: vi.fn(async () => true), + install: vi.fn(async () => '/fake/cua-driver'), + startStatusDaemon: vi.fn(() => daemon), probePermissions: vi.fn(async () => 'ok' as const), - probePermissionStatus: vi.fn(async () => 'ok' as const), + openPermissionPane: vi.fn(), + pollIntervalMs: 1, + pollTimeoutMs: 1000, }; }); @@ -46,11 +54,11 @@ describe('runBootstrap', () => { rmSync(tmpHome, { recursive: true, force: true }); }); - it('starts the client when binary is approved + permissions ok', async () => { + it('starts the proxy directly when already granted (no panes opened)', async () => { const { saveInstallState } = await import('./install-state.js'); await saveInstallState(tmpHome, { - approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0', - approvedAtIso: '2026-05-28T10:00:00Z', + approvedPackageSpec: KEY, + approvedAtIso: '2026-06-12T10:00:00Z', }); const client = makeFakeClient(); @@ -60,8 +68,10 @@ describe('runBootstrap', () => { deps, ); + expect(deps.install).toHaveBeenCalledOnce(); + expect(deps.openPermissionPane).not.toHaveBeenCalled(); + expect(daemon.kill).toHaveBeenCalled(); // status daemon torn down expect(client.start).toHaveBeenCalledOnce(); - expect(deps.promptInstallApproval).not.toHaveBeenCalled(); }); it('prompts for install approval on first call', async () => { @@ -71,15 +81,13 @@ describe('runBootstrap', () => { { signal: new AbortController().signal }, deps, ); - expect(deps.promptInstallApproval).toHaveBeenCalledOnce(); expect(client.start).toHaveBeenCalledOnce(); }); - it('throws when user declines install', async () => { + it('throws and does NOT download when user declines install', async () => { deps.promptInstallApproval = vi.fn(async () => false); const client = makeFakeClient(); - await expect( runBootstrap( client as never, @@ -87,33 +95,26 @@ describe('runBootstrap', () => { deps, ), ).rejects.toThrow(/declined/i); + expect(deps.install).not.toHaveBeenCalled(); expect(client.start).not.toHaveBeenCalled(); }); - it('auto-approves install under YOLO (ctx.autoApproveInstall) without prompting', async () => { - // First use (no install state). In YOLO mode the scheduler bypasses - // the confirmation dialog, so its onConfirm never records approval. - // promptInstallApproval is set to REFUSE here to prove the YOLO path - // skips it entirely rather than coincidentally returning true. - deps.promptInstallApproval = vi.fn(async () => false); - const client = makeFakeClient(); - - await runBootstrap( - client as never, - { signal: new AbortController().signal, autoApproveInstall: true }, - deps, - ); + it('guides one permission at a time: Accessibility pane, then Screen Recording pane', async () => { + const { saveInstallState } = await import('./install-state.js'); + await saveInstallState(tmpHome, { + approvedPackageSpec: KEY, + approvedAtIso: '2026-06-12T10:00:00Z', + }); - // Did not throw "declined", and never consulted the headless prompt. - expect(deps.promptInstallApproval).not.toHaveBeenCalled(); - expect(client.start).toHaveBeenCalledOnce(); - // Approval is persisted so later (interactive) calls skip the prompt too. - const { loadInstallState } = await import('./install-state.js'); - const state = await loadInstallState(tmpHome); - expect(state?.approvedPackageSpec).toBe(deps.packageSpec); - }); + // accessibility missing → screen recording missing → ok + let n = 0; + deps.probePermissions = vi.fn(async () => { + n++; + if (n === 1) return 'accessibility' as const; + if (n === 2) return 'screenRecording' as const; + return 'ok' as const; + }); - it('persists approval on success', async () => { const client = makeFakeClient(); await runBootstrap( client as never, @@ -121,32 +122,26 @@ describe('runBootstrap', () => { deps, ); - const { loadInstallState } = await import('./install-state.js'); - const state = await loadInstallState(tmpHome); - expect(state?.approvedPackageSpec).toBe( - '@qwen-code/open-computer-use@^0.3.0', - ); + const panes = (deps.openPermissionPane as ReturnType).mock + .calls; + expect(panes).toEqual([['accessibility'], ['screenRecording']]); // in order, one each + expect(client.start).toHaveBeenCalledOnce(); }); - it('shows the window once via doctor, then polls the window-free permission-status until granted', async () => { + it('relaunches the status daemon when status reads unknown (e.g. SR restart)', async () => { const { saveInstallState } = await import('./install-state.js'); await saveInstallState(tmpHome, { - approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0', - approvedAtIso: '2026-05-28T10:00:00Z', + approvedPackageSpec: KEY, + approvedAtIso: '2026-06-12T10:00:00Z', }); - // Initial probe (doctor) reports missing → enters the poll loop and - // launches the onboarding window ONCE. - deps.probePermissions = vi.fn(async () => 'accessibility' as const); - // Poll probe (permission-status, window-free) reports missing twice - // then granted. - let pollCount = 0; - deps.probePermissionStatus = vi.fn(async () => { - pollCount++; - return pollCount < 2 ? 'accessibility' : 'ok'; + let n = 0; + deps.probePermissions = vi.fn(async () => { + n++; + if (n === 1) return 'unknown' as const; // daemon coming up / restarted + if (n === 2) return 'accessibility' as const; + return 'ok' as const; }); - deps.pollIntervalMs = 1; // speed up test - deps.pollTimeoutMs = 1000; const client = makeFakeClient(); await runBootstrap( @@ -155,28 +150,19 @@ describe('runBootstrap', () => { deps, ); - // doctor (window) called exactly once; the window-free status command - // is what gets polled — never doctor again (no window storm). - expect(deps.probePermissions).toHaveBeenCalledTimes(1); - expect(pollCount).toBeGreaterThanOrEqual(2); - expect(deps.probePermissionStatus).toHaveBeenCalledWith( - '@qwen-code/open-computer-use@^0.3.0', - ); + // initial launch + one relaunch after 'unknown'. + expect(deps.startStatusDaemon).toHaveBeenCalledTimes(2); + expect(client.start).toHaveBeenCalledOnce(); }); - it('throws after pollTimeoutMs when permissions never grant', async () => { + it('times out (and tears down the daemon) if permissions never arrive', async () => { const { saveInstallState } = await import('./install-state.js'); await saveInstallState(tmpHome, { - approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0', - approvedAtIso: '2026-05-28T10:00:00Z', + approvedPackageSpec: KEY, + approvedAtIso: '2026-06-12T10:00:00Z', }); - - // Initial doctor reports missing (enters loop); window-free poll never - // grants → must time out. deps.probePermissions = vi.fn(async () => 'accessibility' as const); - deps.probePermissionStatus = vi.fn(async () => 'accessibility' as const); - deps.pollIntervalMs = 1; - deps.pollTimeoutMs = 50; + deps.pollTimeoutMs = 30; const client = makeFakeClient(); await expect( @@ -186,13 +172,15 @@ describe('runBootstrap', () => { deps, ), ).rejects.toThrow(/timed out/i); + expect(daemon.kill).toHaveBeenCalled(); + expect(client.start).not.toHaveBeenCalled(); }); - it('skips permission flow on non-darwin platforms', async () => { + it('skips the permission flow on non-darwin platforms', async () => { const { saveInstallState } = await import('./install-state.js'); await saveInstallState(tmpHome, { - approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0', - approvedAtIso: '2026-05-28T10:00:00Z', + approvedPackageSpec: KEY, + approvedAtIso: '2026-06-12T10:00:00Z', }); deps.platform = 'linux'; @@ -202,113 +190,51 @@ describe('runBootstrap', () => { { signal: new AbortController().signal }, deps, ); - + expect(deps.startStatusDaemon).not.toHaveBeenCalled(); expect(deps.probePermissions).not.toHaveBeenCalled(); - expect(deps.probePermissionStatus).not.toHaveBeenCalled(); - }); - - it('emits a fresh updateOutput message when permission kind changes mid-poll', async () => { - const { saveInstallState } = await import('./install-state.js'); - await saveInstallState(tmpHome, { - approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0', - approvedAtIso: '2026-05-28T10:00:00Z', - }); - - // Initial doctor reports accessibility missing (enters loop, window once). - deps.probePermissions = vi.fn(async () => 'accessibility' as const); - // Window-free poll sequence: accessibility → screenRecording → ok - let pollCount = 0; - deps.probePermissionStatus = vi.fn(async () => { - pollCount++; - if (pollCount === 1) return 'screenRecording' as const; - return 'ok' as const; - }); - deps.pollIntervalMs = 1; - deps.pollTimeoutMs = 1000; - - const messages: string[] = []; - const client = makeFakeClient(); - await runBootstrap( - client as never, - { - signal: new AbortController().signal, - updateOutput: (msg) => messages.push(msg), - }, - deps, - ); - - // The transition (accessibility → screenRecording) must emit a - // user-facing message naming the new permission kind. - expect(messages.some((m) => m.includes('screenRecording'))).toBe(true); - expect(messages.some((m) => m.includes('accessibility'))).toBe(true); + expect(client.start).toHaveBeenCalledOnce(); }); - it('skips permission probe when client is already started (no probe spam per tool call)', async () => { - // Regression: bootstrap used to call probePermissions on EVERY - // tool call (which in the previous Finder-based probe popped Finder - // to the foreground each time). The wasAlreadyStarted check makes - // probe fire only on a fresh client start. + it('does nothing extra when the client is already started (warm)', async () => { const { saveInstallState } = await import('./install-state.js'); await saveInstallState(tmpHome, { - approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0', - approvedAtIso: '2026-05-28T10:00:00Z', + approvedPackageSpec: KEY, + approvedAtIso: '2026-06-12T10:00:00Z', }); - - const startSpy = vi.fn(async () => {}); const client = { - isStarted: vi.fn(() => true), // already started - start: startSpy, + isStarted: vi.fn(() => true), + start: vi.fn(async () => {}), + stop: vi.fn(async () => {}), callTool: vi.fn(), - stop: vi.fn(), }; - await runBootstrap( client as never, { signal: new AbortController().signal }, deps, ); - - expect(startSpy).not.toHaveBeenCalled(); - expect(deps.probePermissions).not.toHaveBeenCalled(); - expect(deps.probePermissionStatus).not.toHaveBeenCalled(); + expect(client.start).not.toHaveBeenCalled(); + expect(deps.startStatusDaemon).not.toHaveBeenCalled(); }); }); -describe('parseDoctorStdout', () => { - it("returns 'ok' when doctor reports both permissions granted", () => { - const stdout = - 'Permissions: accessibility=granted, screenRecording=granted\n'; - expect(parseDoctorStdout(stdout)).toBe('ok'); - }); - - it("returns 'accessibility' when accessibility is missing", () => { - const stdout = - 'Permissions: accessibility=missing, screenRecording=granted\n'; - expect(parseDoctorStdout(stdout)).toBe('accessibility'); - }); - - it("returns 'screenRecording' when only Screen Recording is missing", () => { - const stdout = - 'Permissions: accessibility=granted, screenRecording=missing\n'; - expect(parseDoctorStdout(stdout)).toBe('screenRecording'); - }); - - it("prefers 'accessibility' when both are missing (driven by doctor's onboarding order)", () => { - const stdout = - 'Permissions: accessibility=missing, screenRecording=missing\n'; - expect(parseDoctorStdout(stdout)).toBe('accessibility'); - }); - - it('parses case-insensitively and tolerates whitespace around `=`', () => { - const stdout = - 'Permissions: Accessibility = Granted, ScreenRecording = Granted\n'; - expect(parseDoctorStdout(stdout)).toBe('ok'); - }); - - it("returns 'accessibility' when stdout is empty (defensive: treat unknown as missing)", () => { - // Defensive: if doctor produces no parseable output, assume the - // worst (permissions missing) — better to over-prompt than to - // silently proceed and have the tool call fail later. - expect(parseDoctorStdout('')).toBe('accessibility'); +describe('parsePermissionsStatus', () => { + it("returns 'ok' when both grants are true", () => { + expect( + parsePermissionsStatus('{"accessibility":true,"screen_recording":true}'), + ).toBe('ok'); + }); + it("returns 'accessibility' when accessibility is false", () => { + expect( + parsePermissionsStatus('{"accessibility":false,"screen_recording":true}'), + ).toBe('accessibility'); + }); + it("returns 'screenRecording' when only screen recording is false", () => { + expect( + parsePermissionsStatus('{"accessibility":true,"screen_recording":false}'), + ).toBe('screenRecording'); + }); + it("returns 'unknown' for daemon-less / unparseable payloads", () => { + expect(parsePermissionsStatus('{"status":"unknown"}')).toBe('unknown'); + expect(parsePermissionsStatus('not json')).toBe('unknown'); }); }); diff --git a/packages/core/src/tools/computer-use/bootstrap.ts b/packages/core/src/tools/computer-use/bootstrap.ts index 5d977ae9073..34fcfd93045 100644 --- a/packages/core/src/tools/computer-use/bootstrap.ts +++ b/packages/core/src/tools/computer-use/bootstrap.ts @@ -5,211 +5,214 @@ */ /** - * Computer Use bootstrap state machine. + * Computer Use bootstrap state machine (cua-driver backend). * - * On first invocation of any computer_use__* tool: - * 1. If not yet approved: prompt the user to install (one-time). - * 2. Start the client (lazy npx spawn, may take ~60s first time). - * 3. On macOS only: probe permissions via the upstream CLI (NOT via - * get_app_state, which has the side-effect of activating the target - * app — earlier rounds probed Finder this way and caused Finder to - * pop to the foreground at session start). Two distinct commands: - * - `doctor` (initial probe, once): reads TCC + runtime preflight, - * prints "Permissions: accessibility=..., screenRecording=..." - * to stdout, AND launches the onboarding window when any - * permission is missing. This is what shows the window — once. - * - `permission-status` (poll probe): prints the same summary but - * NEVER launches a window. The poll loop uses this so it does - * not spawn a new onboarding window on every iteration. - * `doctor` does NOT dedup its window — each invocation launches a - * fresh one — so polling `doctor` flooded the screen with windows. - * Probing the window-free `permission-status` in the loop fixes that - * while keeping the "grant then auto-continue" UX. - * Requires @qwen-code/open-computer-use >= 0.2.2 for permission-status. + * cua-driver is a persistent daemon (`CuaDriver serve` under com.trycua.driver) + * fronted by a thin `cua-driver mcp` stdio proxy. Tools only work once the + * daemon has BOTH macOS grants (Accessibility + Screen Recording). + * + * First-use permission flow — driven so the user grants ONE permission at a + * time, and so we can reliably detect progress (the two problems with the + * native `permissions grant`: it requests both at once, and while its daemon + * sits in the all-or-nothing gate `permissions status` reports `unknown`, so a + * partial grant is undetectable). Instead: + * + * 1. Run a status-only daemon with `serve --no-permissions-gate` (launched + * via `open -a CuaDriver` so it carries the com.trycua.driver identity). + * With the gate off it SERVES IMMEDIATELY even with no grants, so + * `permissions status --json` returns accurate PER-PERMISSION booleans. + * 2. POLL status every 5s. Open the System Settings pane for whichever + * permission is still missing — Accessibility first, then Screen + * Recording — one at a time, guiding the user. + * 3. Granting Screen Recording force-restarts the daemon → the next poll + * reads `unknown`; we relaunch the status daemon and keep polling. + * 4. Once both are granted, tear the status daemon down and spawn the real + * proxy. Any residual restart is absorbed by the client's reconnect. */ -import { execFile } from 'node:child_process'; +import { execFile, spawnSync } from 'node:child_process'; import { promisify } from 'node:util'; +import { rmSync } from 'node:fs'; import { homedir } from 'node:os'; +import { join } from 'node:path'; import type { ComputerUseClient } from './client.js'; import { isPackageSpecApproved, saveInstallState } from './install-state.js'; -import { type PermissionErrorKind } from './permission-detector.js'; -import { resolveComputerUsePackageSpec } from './constants.js'; +import { approvalKey, binaryPath } from './constants.js'; +import { ensureInstalled } from './downloader.js'; const execFileAsync = promisify(execFile); export interface BootstrapContext { signal: AbortSignal; updateOutput?: (output: string) => void; - /** - * Treat the first-use install as pre-approved, skipping the - * promptInstallApproval gate. Set by the caller when the active approval - * mode auto-approves tool calls and bypasses ComputerUseTool's confirmation - * dialog (YOLO / AUTO_EDIT / AUTO): in those modes the dialog's onConfirm - * never records install approval, so without this flag the headless - * fallback below would refuse and throw "install declined by user". The - * approval is still persisted, so later interactive calls skip the prompt. - */ + /** Treat the first-use install as pre-approved (YOLO / AUTO_EDIT / AUTO). */ autoApproveInstall?: boolean; } -/** Result of a permission probe. */ -export type PermissionProbeResult = 'ok' | PermissionErrorKind; +/** + * Result of a permission probe: + * - 'ok' both grants present + * - 'accessibility' Accessibility missing + * - 'screenRecording' Accessibility present, Screen Recording missing + * - 'unknown' couldn't read status (no daemon yet / restarting) + */ +export type PermissionProbeResult = + | 'ok' + | 'accessibility' + | 'screenRecording' + | 'unknown'; + +/** A running status daemon we can tear down. */ +export interface StatusDaemon { + kill: () => void; +} export interface BootstrapDeps { homeDir: string; - packageSpec: string; + approvalKey: string; platform: NodeJS.Platform; + promptInstallApproval: (key: string) => Promise; + install: (onProgress?: (m: string) => void) => Promise; /** - * Prompt the user to approve installing the upstream binary. Returns - * true if approved. Default uses stderr + the - * QWEN_COMPUTER_USE_AUTO_APPROVE=1 env-var fallback; the interactive - * confirmation dialog is wired through ComputerUseTool's - * getConfirmationDetails(), which runs BEFORE execute() reaches - * runBootstrap (so by the time we get here the install-state file - * already exists for interactive sessions and this fallback is the - * headless / SDK path only). - */ - promptInstallApproval: (packageSpec: string) => Promise; - /** - * Initial probe: runs `doctor`, which both reports status AND launches - * the onboarding window when permissions are missing. Called exactly - * once on a fresh client start so the onboarding window appears one time. - */ - probePermissions: (packageSpec: string) => Promise; - /** - * Poll probe: runs `permission-status`, which reports status WITHOUT - * launching the onboarding window. Called on every poll iteration while - * waiting for the user to grant permissions — using the window-free - * command here is what prevents the onboarding-window storm. + * Launch a status-only daemon (`serve --no-permissions-gate` via + * `open -a CuaDriver`) so `permissions status` returns per-permission + * booleans even before any grant. Returns a handle to tear it down. */ - probePermissionStatus: ( - packageSpec: string, - ) => Promise; - /** Poll interval for the permission watcher. Default 5000ms. */ + startStatusDaemon: () => StatusDaemon; + /** Read current TCC status (`permissions status --json`). */ + probePermissions: () => Promise; + /** Open the System Settings pane for one permission so the user can grant it. */ + openPermissionPane: (kind: 'accessibility' | 'screenRecording') => void; + /** Poll interval. Default 5000ms. */ pollIntervalMs?: number; /** Total poll timeout. Default 10 min. */ pollTimeoutMs?: number; } /** - * Parse the doctor stdout summary into a probe result. - * - * Doctor prints a single line of the form: - * "Permissions: accessibility=granted, screenRecording=missing" - * - * Exported separately from probePermissionsViaDoctor so unit tests can - * exercise the parse logic without spawning a real npx process. + * Parse `cua-driver permissions status --json` into a probe result. + * Shape: `{ accessibility: bool, screen_recording: bool, ... }`. */ -export function parseDoctorStdout(stdout: string): PermissionProbeResult { - const accessibilityGranted = /accessibility\s*=\s*granted/i.test(stdout); - const screenRecordingGranted = /screenrecording\s*=\s*granted/i.test(stdout); - if (!accessibilityGranted) return 'accessibility'; - if (!screenRecordingGranted) return 'screenRecording'; - return 'ok'; +export function parsePermissionsStatus(json: string): PermissionProbeResult { + try { + const o = JSON.parse(json) as { + accessibility?: boolean; + screen_recording?: boolean; + }; + if (typeof o.accessibility !== 'boolean') return 'unknown'; + if (!o.accessibility) return 'accessibility'; + if (!o.screen_recording) return 'screenRecording'; + return 'ok'; + } catch { + return 'unknown'; + } } -/** - * Probe macOS permissions by spawning the upstream doctor CLI. - * - * Doctor runs `PermissionDiagnostics.current()` (reads TCC SQLite + - * runtime preflight via AXIsProcessTrusted() / CGPreflightScreenCaptureAccess()), - * prints the summary to stdout, and — only if any permissions are - * missing — launches the onboarding window via LaunchServices. The - * doctor process exits in both cases. - * - * Key UX property: when permissions are already granted, doctor exits - * silently without opening any window. Unlike the previous get_app_state - * probe, NO target app is activated by the probe itself. - * - * Cost: each invocation spawns `npx`. With the binary cached this is - * ~200-500ms total. Steady-state runs (permissions OK) pay this once - * per fresh client start; the polling loop pays it every pollIntervalMs - * only while permissions are missing (i.e., during initial setup). - * - * Returns: - * - 'ok' → both permissions granted - * - 'accessibility' → Accessibility missing - * - 'screenRecording' → AX granted, Screen Recording missing - * - 'other' → spawn / parse failed; skip probe and let the - * real tool call surface any permission error - */ -export async function probePermissionsViaDoctor( - packageSpec: string, -): Promise { +const SOCKET = () => + join(homedir(), 'Library', 'Caches', 'cua-driver', 'cua-driver.sock'); + +function killServeDaemons(): void { try { - const { stdout } = await execFileAsync( - 'npx', - ['-y', packageSpec, 'doctor'], + spawnSync( + 'pkill', + ['-f', 'CuaDriver.app/Contents/MacOS/cua-driver serve'], { - timeout: 30000, - env: process.env as NodeJS.ProcessEnv, + stdio: 'ignore', }, ); - return parseDoctorStdout(stdout); } catch { - // Spawn failed (npx missing, network down on first run, timeout, etc.) - // OR doctor exited non-zero. Skip probe; the next real tool call - // will surface any permission error via upstream's normal error path. - return 'other'; + // ignore + } + try { + rmSync(SOCKET(), { force: true }); + } catch { + // ignore + } +} + +/** Probe via the window-free `permissions status --json` CLI (non-blocking). */ +export async function probePermissionsViaStatus(): Promise { + try { + const { stdout } = await execFileAsync( + binaryPath(homedir()), + ['permissions', 'status', '--json'], + { timeout: 10_000, env: process.env as NodeJS.ProcessEnv }, + ); + return parsePermissionsStatus(stdout); + } catch { + return 'unknown'; } } /** - * Probe macOS permissions via the `permission-status` CLI command — - * the window-free counterpart to `doctor`. - * - * `permission-status` prints the SAME summary line as `doctor` but NEVER - * launches the onboarding window. This is the probe the polling loop uses - * while waiting for the user to grant permissions: `doctor` re-launches a - * fresh onboarding window on every invocation (it does not dedup), so - * polling it every few seconds floods the screen with windows. We launch - * the window exactly once (the initial `doctor` probe) and then poll this - * window-free command. - * - * Requires `@qwen-code/open-computer-use@>=0.2.2`. On older pinned packages - * the command is unknown → npx exits non-zero → we return 'other', which - * the poll loop treats as non-blocking (exits the wait; the real tool call - * then surfaces any permission error). No window storm either way. + * Launch the status-only daemon. `open -a CuaDriver` gives it the + * com.trycua.driver TCC identity; `--no-permissions-gate` makes it serve + * immediately so status reads work before grants land. Kills any prior daemon + * first so there is exactly one. */ -export async function probePermissionStatusViaCLI( - packageSpec: string, -): Promise { +export function startStatusDaemonProcess(): StatusDaemon { + killServeDaemons(); try { - const { stdout } = await execFileAsync( - 'npx', - ['-y', packageSpec, 'permission-status'], - { - timeout: 30000, - env: process.env as NodeJS.ProcessEnv, - }, + spawnSync( + 'open', + [ + '-n', + '-g', + '-a', + 'CuaDriver', + '--args', + 'serve', + '--no-permissions-gate', + ], + { stdio: 'ignore' }, + ); + } catch { + // ignore — the poll loop reports 'unknown' and retries. + } + return { kill: killServeDaemons }; +} + +/** Open the System Settings privacy pane for a permission. */ +export function openPermissionPaneProcess( + kind: 'accessibility' | 'screenRecording', +): void { + const anchor = + kind === 'accessibility' + ? 'Privacy_Accessibility' + : 'Privacy_ScreenCapture'; + try { + spawnSync( + 'open', + [`x-apple.systempreferences:com.apple.preference.security?${anchor}`], + { stdio: 'ignore' }, ); - return parseDoctorStdout(stdout); } catch { - return 'other'; + // ignore — the message still tells the user where to go. } } /** Production defaults — instantiated lazily so tests can override per call. */ function defaultDeps(): BootstrapDeps { - const packageSpec = resolveComputerUsePackageSpec(); + const home = homedir(); return { - homeDir: homedir(), - packageSpec, + homeDir: home, + approvalKey: approvalKey(), platform: process.platform, - promptInstallApproval: async (spec) => { + promptInstallApproval: async (key) => { process.stderr.write( - `\n[Computer Use] First-time install\n` + - ` Package: ${spec}\n` + - ` This will fetch ~50MB from the npm registry the first time.\n` + - ` Computer Use can click, type, and read your desktop apps.\n` + + `\n[Computer Use] First-time setup\n` + + ` Driver: ${key}\n` + + ` This downloads a ~20MB signed + notarized binary into ~/.qwen/computer-use/.\n` + + ` Computer Use can click, type, and read your desktop apps in the background.\n` + ` On macOS you'll be guided through Accessibility and Screen Recording permissions next.\n` + `Set QWEN_COMPUTER_USE_AUTO_APPROVE=1 to skip this prompt.\n`, ); return process.env['QWEN_COMPUTER_USE_AUTO_APPROVE'] === '1'; }, - probePermissions: probePermissionsViaDoctor, - probePermissionStatus: probePermissionStatusViaCLI, + install: (onProgress) => ensureInstalled({ home, onProgress }), + startStatusDaemon: startStatusDaemonProcess, + probePermissions: probePermissionsViaStatus, + openPermissionPane: openPermissionPaneProcess, }; } @@ -222,18 +225,16 @@ export async function runBootstrap( const pollIntervalMs = deps.pollIntervalMs ?? 5000; const pollTimeoutMs = deps.pollTimeoutMs ?? 10 * 60_000; - // Step 1: install approval gate. - const approved = await isPackageSpecApproved(deps.homeDir, deps.packageSpec); + // Step 1: install approval gate (gates the download). + const approved = await isPackageSpecApproved(deps.homeDir, deps.approvalKey); if (!approved) { if (ctx.autoApproveInstall) { - // An auto-approve mode (YOLO / AUTO_EDIT / AUTO) already approved the - // tool call and bypassed the confirmation dialog whose onConfirm would - // have recorded approval, so honor that intent here instead of falling - // through to the headless prompt (which refuses and throws). ctx.updateOutput?.('Computer Use install auto-approved (approval mode).'); } else { - ctx.updateOutput?.('Computer Use needs to be installed (first use).'); - const ok = await deps.promptInstallApproval(deps.packageSpec); + ctx.updateOutput?.( + 'Computer Use needs a one-time driver download (first use).', + ); + const ok = await deps.promptInstallApproval(deps.approvalKey); if (!ok) { throw new Error( `Computer Use install declined by user. Re-invoke the tool to be prompted again.`, @@ -241,84 +242,106 @@ export async function runBootstrap( } } await saveInstallState(deps.homeDir, { - approvedPackageSpec: deps.packageSpec, + approvedPackageSpec: deps.approvalKey, approvedAtIso: new Date().toISOString(), }); } - // Step 2: spawn (idempotent). Remember whether THIS call performed - // the spawn — used below to decide whether to re-probe permissions. - const wasAlreadyStarted = client.isStarted(); - if (!wasAlreadyStarted) { - await client.start(ctx.updateOutput); - } + // Step 2: ensure the binary is present (download on first use; no-op after). + await deps.install(ctx.updateOutput); - // Step 3: macOS permission probe + guide. - // - // Only probe on a fresh client start. Once the upstream binary is - // running with permissions verified, TCC state is stable for the - // process lifetime — re-probing on every tool call would needlessly - // spawn extra doctor processes. - // - // Trade-off on mid-session permission revocation: upstream returns - // permissionDenied as an MCP result with isError=true (not a thrown - // exception), so it does NOT trigger client.callTool's transport- - // closed retry path, and the reconnect path itself goes through - // client.stop() + client.start() directly without re-entering - // runBootstrap. The model therefore receives permissionDenied on - // every subsequent tool call with no automatic recovery — the user - // must restart qwen-code to re-enter the permission flow. This is - // an acceptable trade-off: TCC revocation mid-session is extremely - // rare. - if (wasAlreadyStarted) return; - if (deps.platform !== 'darwin') return; + // A warm client (already started this session) is past the permission flow. + if (client.isStarted()) return; - const probe = await deps.probePermissions(deps.packageSpec); - if (probe === 'ok' || probe === 'other') { - // 'other' means doctor failed for an unexpected reason; we don't - // block bootstrap on that — let the actual tool call surface it. - return; + // Step 3: macOS permission flow (one permission at a time; see file header). + if (deps.platform === 'darwin') { + await ensurePermissions(deps, ctx, pollIntervalMs, pollTimeoutMs); } - // probe == 'accessibility' | 'screenRecording' | 'unknown_permission': - // doctor has ALREADY launched the onboarding window from its own - // process. We just inform the user and enter the poll loop. - ctx.updateOutput?.( - `Computer Use needs macOS permissions (${probe}). ` + - `The onboarding window is opening — please grant Accessibility and Screen Recording, then this will continue automatically.`, - ); + // Step 4: spawn the proxy against the now-granted daemon. + await client.start(ctx.updateOutput); +} - // Track the last probe kind so we can emit a fresh message on - // transition (e.g. accessibility → screenRecording). The onboarding - // window was launched once by the initial `doctor` probe above; the - // poll loop below uses the window-free `permission-status` command so - // it never spawns additional windows while the single onboarding - // window stays open. - let lastProbeKind: PermissionProbeResult = probe; +async function ensurePermissions( + deps: BootstrapDeps, + ctx: BootstrapContext, + pollIntervalMs: number, + pollTimeoutMs: number, +): Promise { + // A status-only (no-gate) daemon so `permissions status` reports per- + // permission booleans throughout — this is what makes partial grants + // detectable and lets us guide one permission at a time. + let daemon = deps.startStatusDaemon(); + let openedAccessibility = false; + let openedScreenRecording = false; - const startedAt = Date.now(); - for (;;) { - if (ctx.signal.aborted) { - throw new Error('Computer Use bootstrap aborted.'); - } - if (Date.now() - startedAt > pollTimeoutMs) { - throw new Error( - `Computer Use permission grant timed out after ${Math.round(pollTimeoutMs / 1000)}s. Re-invoke the tool to retry.`, - ); - } - await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); - // Window-free status check — see probePermissionStatusViaCLI. - const next = await deps.probePermissionStatus(deps.packageSpec); - if (next === 'ok' || next === 'other') return; + try { + const startedAt = Date.now(); + for (;;) { + if (ctx.signal.aborted) + throw new Error('Computer Use bootstrap aborted.'); + if (Date.now() - startedAt > pollTimeoutMs) { + throw new Error( + `Computer Use permission grant timed out after ${Math.round( + pollTimeoutMs / 1000, + )}s. Re-invoke the tool to retry.`, + ); + } + await sleep(pollIntervalMs); + const probe = await deps.probePermissions(); - if (next !== lastProbeKind) { - ctx.updateOutput?.( - `Now waiting for ${next} permission. The onboarding window remains open — please grant this permission to continue.`, - ); - lastProbeKind = next; - } + if (probe === 'ok') return; + + if (probe === 'unknown') { + // No serving daemon — first launch still coming up, or the daemon was + // restarted by a Screen-Recording grant. Relaunch and keep polling. + daemon.kill(); + daemon = deps.startStatusDaemon(); + const elapsed = Math.round((Date.now() - startedAt) / 1000); + ctx.updateOutput?.( + `Bringing up Computer Use permissions check… (${elapsed}s)`, + ); + continue; + } - const elapsedSec = Math.round((Date.now() - startedAt) / 1000); - ctx.updateOutput?.(`Waiting for ${next} permission... (${elapsedSec}s)`); + if (probe === 'accessibility') { + if (!openedAccessibility) { + openedAccessibility = true; + deps.openPermissionPane('accessibility'); + ctx.updateOutput?.( + 'Step 1/2 — In the System Settings window that opened ' + + '(Privacy & Security → Accessibility), turn ON CuaDriver. ' + + 'This continues automatically.', + ); + } else { + const elapsed = Math.round((Date.now() - startedAt) / 1000); + ctx.updateOutput?.( + `Waiting for Accessibility… (${elapsed}s) — enable CuaDriver in System Settings.`, + ); + } + continue; + } + + // probe === 'screenRecording' + if (!openedScreenRecording) { + openedScreenRecording = true; + deps.openPermissionPane('screenRecording'); + ctx.updateOutput?.( + 'Step 2/2 — Accessibility granted. Now in System Settings ' + + '(Privacy & Security → Screen & System Audio Recording), turn ON ' + + 'CuaDriver. macOS will ask to restart CuaDriver — allow it; that is ' + + 'expected. This continues automatically.', + ); + } else { + const elapsed = Math.round((Date.now() - startedAt) / 1000); + ctx.updateOutput?.( + `Waiting for Screen Recording… (${elapsed}s) — enable CuaDriver in System Settings.`, + ); + } + } + } finally { + daemon.kill(); } } + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); diff --git a/packages/core/src/tools/computer-use/client.test.ts b/packages/core/src/tools/computer-use/client.test.ts index 6cdaabe8bd5..91fbc89f392 100644 --- a/packages/core/src/tools/computer-use/client.test.ts +++ b/packages/core/src/tools/computer-use/client.test.ts @@ -5,7 +5,7 @@ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; describe('ComputerUseClient', () => { it('is constructible', () => { const client = new ComputerUseClient({ - packageSpec: '@qwen-code/open-computer-use@latest', + binary: '/fake/cua-driver', onProgress: vi.fn(), }); expect(client).toBeDefined(); @@ -13,7 +13,7 @@ describe('ComputerUseClient', () => { it('reports not-started before start() is called', () => { const client = new ComputerUseClient({ - packageSpec: '@qwen-code/open-computer-use@latest', + binary: '/fake/cua-driver', onProgress: vi.fn(), }); expect(client.isStarted()).toBe(false); @@ -44,6 +44,25 @@ describe('isTransportClosedError', () => { expect(isTransportClosedError(new Error('Not connected'))).toBe(true); }); + it('matches the daemon-restart error (Screen Recording grant → daemon restart)', () => { + // The first-use failure mode: after granting Screen Recording, macOS + // restarts the CuaDriver daemon; the proxy → daemon Unix socket dies. + expect( + isTransportClosedError( + new Error( + 'MCP error -32603: daemon transport error forwarding `list_windows`: connect to /Users/x/Library/Caches/cua-driver/cua-driver.sock: Connection refused (os error 61)', + ), + ), + ).toBe(true); + }); + + it('matches a bare "Connection refused" / "os error 61"', () => { + expect(isTransportClosedError(new Error('Connection refused'))).toBe(true); + expect( + isTransportClosedError(new Error('connect failed: os error 61')), + ).toBe(true); + }); + it('is case-insensitive', () => { expect(isTransportClosedError(new Error('connection closed'))).toBe(true); expect(isTransportClosedError(new Error('NOT CONNECTED'))).toBe(true); @@ -129,7 +148,7 @@ class ReconnectTestClient extends ComputerUseClient { function makeClient(): ReconnectTestClient { const c = new ReconnectTestClient({ - packageSpec: '@qwen-code/open-computer-use@latest', + binary: '/fake/cua-driver', }); // Pre-seed the started state so callTool guard passes. (c as unknown as { client: object }).client = { __fake: true }; @@ -203,6 +222,24 @@ describe('callTool reconnect path', () => { expect(c.startCalled).toBe(1); }); + it('reconnects on the daemon-restart "Connection refused" error', async () => { + const c = makeClient(); + c.behaviors = [ + async () => { + throw new Error( + 'MCP error -32603: daemon transport error forwarding `list_windows`: connect to /Users/x/Library/Caches/cua-driver/cua-driver.sock: Connection refused (os error 61)', + ); + }, + async () => successResult, + ]; + + const result = await c.callTool('list_windows', { pid: 717 }); + + expect(result).toBe(successResult); + expect(c.stopCalled).toBe(1); + expect(c.startCalled).toBe(1); + }); + it('does NOT reconnect on non-transport errors (e.g. upstream tool validation)', async () => { const c = makeClient(); c.behaviors = [ diff --git a/packages/core/src/tools/computer-use/client.ts b/packages/core/src/tools/computer-use/client.ts index 3f5960325c6..ee9359b6fad 100644 --- a/packages/core/src/tools/computer-use/client.ts +++ b/packages/core/src/tools/computer-use/client.ts @@ -10,24 +10,26 @@ import type { CallToolResult, ListToolsResult, } from '@modelcontextprotocol/sdk/types.js'; -import { resolveComputerUsePackageSpec } from './constants.js'; +import { homedir } from 'node:os'; +import { binaryPath } from './constants.js'; /** - * Singleton stdio MCP client for the upstream open-computer-use binary. + * Singleton stdio MCP client for the cua-driver binary. * - * Spawned via `npx -y mcp`. First spawn pays the npx - * download cost (up to ~60s for a fresh cache); subsequent spawns reuse - * the npx cache and are sub-second. + * Spawned via ` mcp`, where `` is the pinned cua-driver + * downloaded under `~/.qwen/computer-use/` (the bootstrap state machine + * downloads + verifies it before the first spawn). Spawns are sub-second + * — there is no npx/download cost on this path anymore. * * Lifecycle: lazy spawn on first `callTool` invocation. The process * stays alive until `stop()` or qwen-code exits. State (element_index - * map per app) lives in the process — if the process restarts, the - * model must call `get_app_state` again before any element-targeted + * map per window) lives in the process — if the process restarts, the + * model must call `get_window_state` again before any element-targeted * action. */ export interface ComputerUseClientOptions { - /** npm package spec to npx. Example: "@qwen-code/open-computer-use@0.2.3". */ - packageSpec: string; + /** Absolute path to the spawnable `cua-driver` binary. */ + binary: string; /** Streaming hook for progress messages during slow operations. */ onProgress?: (message: string) => void; } @@ -35,30 +37,28 @@ export interface ComputerUseClientOptions { export class ComputerUseClient { private static singleton: ComputerUseClient | undefined; - private readonly packageSpec: string; + private readonly binary: string; private readonly onProgress: (message: string) => void; private client: Client | undefined; private startPromise: Promise | undefined; constructor(options: ComputerUseClientOptions) { - this.packageSpec = options.packageSpec; + this.binary = options.binary; this.onProgress = options.onProgress ?? (() => {}); } /** * Shared singleton instance, created with default options on first * access. Tests can replace it via `setSharedForTest()`. + * + * The binary path is derived from the pinned `CUA_DRIVER_VERSION` in + * constants.ts, the single source of truth the downloaded binary + + * generated `schemas.ts` agree on. */ static shared(): ComputerUseClient { if (!ComputerUseClient.singleton) { - // Use the single source of truth for the package spec - // (PINNED_OPEN_COMPUTER_USE_VERSION in constants.ts). The previous - // inline `?? 'open-computer-use@latest'` fallback meant the actual - // MCP server could run a newer upstream than the schemas.ts pin - // was generated against — DragonnZhang flagged the schema-drift - // window in PR review. ComputerUseClient.singleton = new ComputerUseClient({ - packageSpec: resolveComputerUsePackageSpec(), + binary: binaryPath(homedir()), }); } return ComputerUseClient.singleton; @@ -97,31 +97,20 @@ export class ComputerUseClient { private async doStart(onProgress?: (message: string) => void): Promise { const progress = onProgress ?? this.onProgress; - progress('Starting Computer Use...'); - - // After ~3s, surface a hint that the slow path is download. - const downloadHintTimer = setTimeout(() => { - progress( - 'Downloading Computer Use binary (this can take ~60s on first use)...', - ); - }, 3000); + progress('Starting Computer Use driver...'); - try { - const transport = new StdioClientTransport({ - command: 'npx', - args: ['-y', this.packageSpec, 'mcp'], - // Inherit env so HTTPS_PROXY etc. flow through to npx - env: { ...process.env } as Record, - }); - const client = new Client( - { name: 'qwen-code-computer-use', version: '1.0.0' }, - { capabilities: {} }, - ); - await client.connect(transport); - this.client = client; - } finally { - clearTimeout(downloadHintTimer); - } + const transport = new StdioClientTransport({ + command: this.binary, + args: ['mcp'], + // Inherit env so HTTPS_PROXY / cua-driver config env flow through. + env: { ...process.env } as Record, + }); + const client = new Client( + { name: 'qwen-code-computer-use', version: '1.0.0' }, + { capabilities: {} }, + ); + await client.connect(transport); + this.client = client; } /** @@ -156,24 +145,37 @@ export class ComputerUseClient { })) as CallToolResult; } catch (err) { if (!isTransportClosedError(err)) throw err; - // Reconnect: upstream binary is commonly killed by macOS after the - // user grants Screen Recording (a TCC restart prompt). The child - // process is dead but the user's task is mid-flight. Transparent - // reconnect + single retry keeps the model's flow uninterrupted. + // The connection died. Two recoverable causes, both fixed by respawning + // the proxy (which relaunches the cua-driver daemon): + // 1. stdio "Connection closed" — the `cua-driver mcp` child was killed. + // 2. "daemon transport error … Connection refused" — the CuaDriver + // DAEMON behind the proxy restarted. macOS forces a restart right + // after the Screen Recording grant, so the proxy's Unix socket to + // the daemon goes dead and every subsequent tool fails. This is the + // first-use failure mode (grant SR → restart → all tools error). // - // Element index state lives in the upstream process and is therefore - // lost across the restart. The model is already instructed (via - // schema descriptions) to call get_app_state before any - // element-targeted action — if its retry uses a stale element_index - // it will get a normal upstream error ("element_index out of range") - // and naturally re-snapshot. - await this.stop(); - await this.start(); - if (!this.client) throw new Error('ComputerUseClient reconnect failed'); - return (await this.client.callTool({ - name, - arguments: args, - })) as CallToolResult; + // Respawn + retry, with a few attempts to absorb the daemon's restart / + // startup window (a single retry can land before the new daemon is up). + // Element-index state is lost across the restart; the model re-snapshots + // via get_window_state on a stale-index error. + let lastErr: unknown = err; + for (let attempt = 0; attempt < 3; attempt++) { + await this.stop(); + await this.start(); + if (!this.client) throw new Error('ComputerUseClient reconnect failed'); + try { + return (await this.client.callTool({ + name, + arguments: args, + })) as CallToolResult; + } catch (retryErr) { + if (!isTransportClosedError(retryErr)) throw retryErr; + lastErr = retryErr; + // Daemon may still be coming up after a restart — back off, retry. + await new Promise((r) => setTimeout(r, 1000)); + } + } + throw lastErr; } } @@ -192,15 +194,20 @@ export class ComputerUseClient { } /** - * Returns true when `err` indicates the MCP transport closed unexpectedly - * (e.g. the upstream child process was killed by macOS after a TCC permission - * grant). The patterns below cover all observed SDK error messages: + * Returns true when `err` indicates a recoverable connection failure — either + * the stdio transport to the `cua-driver mcp` proxy closed, OR the proxy's + * Unix-socket link to the CuaDriver daemon died (daemon restart). Both are + * fixed by respawning the proxy. Observed SDK / cua-driver messages: * - * "Connection closed" – StdioClientTransport stream closed - * "MCP error -32000: ..." – JSON-RPC internal error, often wraps the above - * "Not connected" – Client.callTool guard before transport is open + * "Connection closed" – StdioClientTransport stream closed + * "Not connected" – Client guard before transport is open + * "daemon transport error …" – proxy → daemon Unix socket forward failed + * "Connection refused (os error 61)" – daemon not listening (restarted/down) + * "MCP error -32603 / -32000: …" – JSON-RPC wrapper around the above */ export function isTransportClosedError(err: unknown): boolean { const msg = err instanceof Error ? err.message : String(err); - return /connection closed|not connected/i.test(msg); + return /connection closed|not connected|connection refused|daemon transport error|os error 61/i.test( + msg, + ); } diff --git a/packages/core/src/tools/computer-use/constants.test.ts b/packages/core/src/tools/computer-use/constants.test.ts index 8fd08de66d6..030aa4e1f40 100644 --- a/packages/core/src/tools/computer-use/constants.test.ts +++ b/packages/core/src/tools/computer-use/constants.test.ts @@ -1,74 +1,135 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect } from 'vitest'; +import { join } from 'node:path'; import { - PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME, - PINNED_OPEN_COMPUTER_USE_VERSION, - resolveComputerUsePackageSpec, + CUA_DRIVER_VERSION, + approvalKey, + binaryPath, + resolveAssetTarget, + resolveAssetUrls, + resolveChecksumUrls, } from './constants.js'; -describe('computer-use constants', () => { - let originalEnv: string | undefined; +describe('CUA_DRIVER_VERSION', () => { + it('is an exact semver pin (no range / latest)', () => { + expect(CUA_DRIVER_VERSION).toMatch(/^\d+\.\d+\.\d+$/); + for (const bad of ['latest', 'next', '*', '^', '~']) { + expect(CUA_DRIVER_VERSION).not.toContain(bad); + } + }); +}); - beforeEach(() => { - originalEnv = process.env['QWEN_COMPUTER_USE_PACKAGE']; - delete process.env['QWEN_COMPUTER_USE_PACKAGE']; +describe('resolveAssetTarget', () => { + it('maps darwin/arm64 to the .app-bearing tarball, spawning the in-bundle binary', () => { + const t = resolveAssetTarget('darwin', 'arm64'); + expect(t.asset).toBe( + `cua-driver-rs-${CUA_DRIVER_VERSION}-darwin-arm64.tar.gz`, + ); + // In-bundle binary so cua-driver's TCC auto-relaunch fires (com.trycua.driver). + expect(t.binaryRelPath).toBe('CuaDriver.app/Contents/MacOS/cua-driver'); + expect(t.hasApp).toBe(true); }); - afterEach(() => { - if (originalEnv === undefined) { - delete process.env['QWEN_COMPUTER_USE_PACKAGE']; - } else { - process.env['QWEN_COMPUTER_USE_PACKAGE'] = originalEnv; - } + it('maps darwin/x64 to the x86_64 tarball', () => { + expect(resolveAssetTarget('darwin', 'x64').asset).toBe( + `cua-driver-rs-${CUA_DRIVER_VERSION}-darwin-x86_64.tar.gz`, + ); }); - describe('PINNED_OPEN_COMPUTER_USE_VERSION', () => { - it('is an exact version (no range modifiers)', () => { - // Regression guard: the pin is an exact version, NOT `^x.y.z`, - // NOT `~x.y.z`, NOT `latest`, NOT `*`. Locking the schema surface - // requires an exact pin — upstream is 0.x and may ship - // schema-affecting patches. - expect(PINNED_OPEN_COMPUTER_USE_VERSION).toMatch(/^\d+\.\d+\.\d+$/); - }); + it('maps linux/x64 to the -binary tarball whose lone cua-driver sits at the archive root', () => { + const t = resolveAssetTarget('linux', 'x64'); + // Upstream ships the bare-binary tarball for Linux; it expands to a lone + // `cua-driver` at the root, so there is no wrapper dir (extractDir '.'). + expect(t.asset).toBe( + `cua-driver-rs-${CUA_DRIVER_VERSION}-linux-x86_64-binary.tar.gz`, + ); + expect(t.extractDir).toBe('.'); + expect(t.binaryRelPath).toBe('cua-driver'); + expect(t.hasApp).toBe(false); + }); - it('does not contain dist-tags or wildcards', () => { - expect(PINNED_OPEN_COMPUTER_USE_VERSION).not.toContain('latest'); - expect(PINNED_OPEN_COMPUTER_USE_VERSION).not.toContain('next'); - expect(PINNED_OPEN_COMPUTER_USE_VERSION).not.toContain('*'); - expect(PINNED_OPEN_COMPUTER_USE_VERSION).not.toContain('^'); - expect(PINNED_OPEN_COMPUTER_USE_VERSION).not.toContain('~'); - }); + it('maps win32/x64 to the .zip with .exe binary', () => { + const t = resolveAssetTarget('win32', 'x64'); + expect(t.asset).toBe( + `cua-driver-rs-${CUA_DRIVER_VERSION}-windows-x86_64.zip`, + ); + expect(t.binaryRelPath).toBe('cua-driver.exe'); }); - describe('PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME', () => { - it('is the scoped QwenLM fork package', () => { - expect(PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME).toBe( - '@qwen-code/open-computer-use', - ); - }); + it('throws on unsupported platforms / arches', () => { + expect(() => resolveAssetTarget('linux', 'arm64')).toThrow(/unsupported/i); + expect(() => resolveAssetTarget('aix' as never, 'x64')).toThrow( + /unsupported/i, + ); }); +}); - describe('resolveComputerUsePackageSpec', () => { - it('defaults to @ when env var is unset', () => { - expect(resolveComputerUsePackageSpec()).toBe( - `${PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME}@${PINNED_OPEN_COMPUTER_USE_VERSION}`, - ); - }); +describe('resolveAssetUrls', () => { + it('orders sources OSS-first, GitHub-fallback by default', () => { + const urls = resolveAssetUrls('a.tar.gz', {}); + expect(urls).toHaveLength(2); + expect(urls[0]).toContain('aliyuncs.com'); + expect(urls[0]).toContain(`/cua-driver-rs/v${CUA_DRIVER_VERSION}/a.tar.gz`); + expect(urls[1]).toContain('github.com/trycua/cua/releases/download'); + }); - it('honors QWEN_COMPUTER_USE_PACKAGE override', () => { - process.env['QWEN_COMPUTER_USE_PACKAGE'] = - '@qwen-code/open-computer-use@0.99.99'; - expect(resolveComputerUsePackageSpec()).toBe( - '@qwen-code/open-computer-use@0.99.99', - ); + it('prepends QWEN_COMPUTER_USE_DOWNLOAD_HOST as the first source', () => { + const urls = resolveAssetUrls('a.tar.gz', { + QWEN_COMPUTER_USE_DOWNLOAD_HOST: 'https://mirror.internal/', }); + expect(urls).toHaveLength(3); + expect(urls[0]).toBe( + `https://mirror.internal/cua-driver-rs/v${CUA_DRIVER_VERSION}/a.tar.gz`, + ); + }); - it('reads env var at call time (not at module load)', () => { - // Different overrides between calls should both be picked up — - // tests that mutate the env var must see fresh values per call. - process.env['QWEN_COMPUTER_USE_PACKAGE'] = 'spec-a'; - expect(resolveComputerUsePackageSpec()).toBe('spec-a'); - process.env['QWEN_COMPUTER_USE_PACKAGE'] = 'spec-b'; - expect(resolveComputerUsePackageSpec()).toBe('spec-b'); - }); + it('checksum URLs follow the same source order', () => { + const urls = resolveChecksumUrls({}); + expect(urls[0]).toContain('checksums.txt'); + expect(urls[1]).toContain('github.com'); + }); +}); + +describe('binaryPath', () => { + it('resolves to the in-bundle binary under ~/.qwen/computer-use/...', () => { + const p = binaryPath('/home/u', 'darwin', 'arm64'); + expect(p).toBe( + join( + '/home/u', + '.qwen', + 'computer-use', + `cua-driver-rs-${CUA_DRIVER_VERSION}`, + `cua-driver-rs-${CUA_DRIVER_VERSION}-darwin-arm64`, + 'CuaDriver.app', + 'Contents', + 'MacOS', + 'cua-driver', + ), + ); + }); + + it('resolves the Linux binary at the version-dir root (no wrapper dir)', () => { + const p = binaryPath('/home/u', 'linux', 'x64'); + expect(p).toBe( + join( + '/home/u', + '.qwen', + 'computer-use', + `cua-driver-rs-${CUA_DRIVER_VERSION}`, + 'cua-driver', + ), + ); + }); +}); + +describe('approvalKey', () => { + it('encodes the pinned version so a bump forces re-approval', () => { + expect(approvalKey()).toBe(`cua-driver-rs@${CUA_DRIVER_VERSION}`); + expect(approvalKey('9.9.9')).toBe('cua-driver-rs@9.9.9'); }); }); diff --git a/packages/core/src/tools/computer-use/constants.ts b/packages/core/src/tools/computer-use/constants.ts index 510c3536476..ebd72d5813e 100644 --- a/packages/core/src/tools/computer-use/constants.ts +++ b/packages/core/src/tools/computer-use/constants.ts @@ -5,52 +5,202 @@ */ /** - * The npm package that provides the Computer Use MCP server. + * Computer Use is backed by `cua-driver` (the Rust implementation, + * `cua-driver-rs`) from trycua/cua — a background, no-focus-stealing + * native automation driver that speaks MCP over stdio (`cua-driver mcp`). * - * This is the QwenLM fork (`@qwen-code/open-computer-use`), not upstream - * `open-computer-use`. The fork rebrands the macOS bundle id, adds the - * `OPEN_COMPUTER_USE_IMAGE_*` screenshot env overrides, and strips - * Codex-specific install paths. Source: - * https://github.com/QwenLM/open-computer-use + * Unlike the previous open-computer-use backend, cua-driver is NOT on npm. + * It ships as per-platform, Developer-ID-signed + Apple-notarized binaries + * attached to GitHub releases (tag `cua-driver-rs-v`). We download + * the pinned asset once into `~/.qwen/computer-use/`, preferring a + * qwen-code-owned OSS mirror (reliable in CN where GitHub release downloads + * are slow/blocked) and falling back to GitHub. * - * NOTE: only the npm *package* name is scoped. The CLI binary the - * package installs is still named `open-computer-use`, and the binary's - * own error strings (e.g. "Run `open-computer-use doctor`") use that - * binary name — `permission-detector.ts` matches against those and must - * NOT be re-scoped. + * Source: https://github.com/trycua/cua/tree/main/libs/cua-driver + * License: MIT (the driver pulls no AGPL deps — AGPL only affects the + * separate `cua-agent[omni]` layer, which we do not consume). */ -export const PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME = - '@qwen-code/open-computer-use'; + +import { join } from 'node:path'; +import { homedir } from 'node:os'; /** - * The exact `@qwen-code/open-computer-use` version this release of - * qwen-code is pinned to. Hardcoded `schemas.ts` is generated against - * this version; bumping it requires re-running the sync script. + * The exact `cua-driver-rs` release this build of qwen-code is pinned to. + * Hardcoded `schemas.ts` is generated against this version. + * + * Exact pin (NOT a range) is deliberate: cua-driver is pre-1.0 and ships + * multiple releases per day, some schema-affecting. Locking the version + * means users get the exact surface we tested; a new upstream release + * can't silently drift our hardcoded schemas or break the download. * - * To bump: - * 1. Update this constant to the new version (e.g. '0.2.1'). - * 2. Run `npx tsx scripts/sync-computer-use-schemas.ts` from the - * repo root — it reads this constant by default. - * 3. Verify the regenerated `schemas.ts` diff is what you expect - * (parameter types, required fields, descriptions). - * 4. Manually smoke-test the e2e flow on macOS. + * To bump: update this, re-run `scripts/sync-computer-use-schemas.ts` + * against the new binary, sync the new assets to OSS via + * `scripts/sync-cua-driver-to-oss.ts`, then smoke-test on macOS. + */ +export const CUA_DRIVER_VERSION = '0.5.2'; + +/** + * qwen-code-owned OSS mirror base (primary download source). Assets live + * under `/cua-driver-rs/v/`. Populated by + * `scripts/sync-cua-driver-to-oss.ts` on each version bump. * - * Using an exact pin (NOT `^x.y.z` or `@latest`) is deliberate: - * the fork is 0.x and may ship schema-affecting changes in a patch - * release. Locking the version means users get the exact schema - * surface we tested against; a new release can't silently drift our - * hardcoded schemas out of sync. + * TODO(release): point at the real qwen-code OSS bucket before shipping. */ -export const PINNED_OPEN_COMPUTER_USE_VERSION = '0.2.3'; +export const OSS_MIRROR_BASE = + 'https://qwen-code.oss-cn-hangzhou.aliyuncs.com/computer-use'; + +/** GitHub release download base for the pinned tag (fallback source). */ +export const GITHUB_RELEASE_BASE = + 'https://github.com/trycua/cua/releases/download'; + +export interface AssetTarget { + /** Release asset filename. */ + asset: string; + /** Directory the tarball/zip extracts into. */ + extractDir: string; + /** Path to the spawnable driver binary, relative to the extract dir. */ + binaryRelPath: string; + /** Whether this asset bundles `CuaDriver.app` (macOS TCC onboarding). */ + hasApp: boolean; +} /** - * Resolve the package spec to `npx` for spawning the MCP server. Reads - * `QWEN_COMPUTER_USE_PACKAGE` env var at call time so tests / power - * users can override the pinned package or version. + * Map a Node platform/arch to the cua-driver release asset. + * Throws for unsupported targets so callers fail loudly rather than + * spawning a missing binary. */ -export function resolveComputerUsePackageSpec(): string { - return ( - process.env['QWEN_COMPUTER_USE_PACKAGE'] ?? - `${PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME}@${PINNED_OPEN_COMPUTER_USE_VERSION}` +export function resolveAssetTarget( + platform: NodeJS.Platform = process.platform, + arch: string = process.arch, + version: string = CUA_DRIVER_VERSION, +): AssetTarget { + const v = version; + if (platform === 'darwin') { + const slug = arch === 'arm64' ? 'darwin-arm64' : 'darwin-x86_64'; + const extractDir = `cua-driver-rs-${v}-${slug}`; + return { + asset: `${extractDir}.tar.gz`, + extractDir, + // Spawn the binary INSIDE CuaDriver.app, not the bare one beside it. + // cua-driver only triggers its TCC auto-relaunch (`open -a CuaDriver + // serve`, which attributes Accessibility/Screen-Recording grants to + // com.trycua.driver rather than the launching terminal) when its + // running image resolves into `/CuaDriver.app/Contents/MacOS/` + // (see bundle.rs `is_executable_inside_cuadriver_app`). Pointing at + // the bare `cua-driver` made TCC attribute to the parent terminal + // (e.g. iTerm) — wrong identity, per-terminal, oversized privacy. + binaryRelPath: 'CuaDriver.app/Contents/MacOS/cua-driver', + hasApp: true, + }; + } + if (platform === 'linux') { + if (arch !== 'x64') { + throw new Error( + `Computer Use: unsupported Linux arch '${arch}' (only x64).`, + ); + } + // Linux ships a BARE-BINARY tarball whose single `cua-driver` file sits at + // the archive ROOT — no bundle, no wrapper dir. Upstream _install-rust.sh + // picks `darwin-universal.tar.gz` (a dir tarball carrying CuaDriver.app) + // for macOS but `${label}-binary.tar.gz` for every other target, which + // expands to a lone `cua-driver`. So there is NO extractDir layer here: + // extractDir '.' keeps binaryPath at /cua-driver. + return { + asset: `cua-driver-rs-${v}-linux-x86_64-binary.tar.gz`, + extractDir: '.', + binaryRelPath: 'cua-driver', + hasApp: false, + }; + } + if (platform === 'win32') { + // Windows uses the NON-binary `.zip` (verified against upstream + // install.ps1: `$zipName = "cua-driver-rs-$version-$archLabel.zip"`), which + // expands to `cua-driver-rs--/cua-driver.exe (+ LICENSE)` — a + // wrapper dir, UNLIKE Linux. This asset mapping is correct; the only gap is + // zip extraction in downloader.ts (node `tar` reads .tar.gz only). + const slug = arch === 'arm64' ? 'windows-arm64' : 'windows-x86_64'; + const extractDir = `cua-driver-rs-${v}-${slug}`; + return { + asset: `${extractDir}.zip`, + extractDir, + binaryRelPath: 'cua-driver.exe', + hasApp: false, + }; + } + throw new Error(`Computer Use: unsupported platform '${platform}'.`); +} + +/** + * Ordered list of full download URLs for an asset: env override (if set), + * then OSS mirror, then GitHub. The downloader tries each in order until + * one succeeds. + * + * `QWEN_COMPUTER_USE_DOWNLOAD_HOST` lets enterprises / power users point at + * an internal mirror laid out like OSS (`/cua-driver-rs/v/`). + */ +export function resolveAssetUrls( + asset: string, + env: NodeJS.ProcessEnv = process.env, + version: string = CUA_DRIVER_VERSION, +): string[] { + const urls: string[] = []; + const override = env['QWEN_COMPUTER_USE_DOWNLOAD_HOST']; + if (override) { + urls.push(`${trimSlash(override)}/cua-driver-rs/v${version}/${asset}`); + } + urls.push(`${OSS_MIRROR_BASE}/cua-driver-rs/v${version}/${asset}`); + urls.push(`${GITHUB_RELEASE_BASE}/cua-driver-rs-v${version}/${asset}`); + return urls; +} + +/** URL for the release `checksums.txt` (same source order as assets). */ +export function resolveChecksumUrls( + env: NodeJS.ProcessEnv = process.env, + version: string = CUA_DRIVER_VERSION, +): string[] { + return resolveAssetUrls('checksums.txt', env, version); +} + +/** Install root for all Computer Use artifacts. Footprint stays here. */ +export function computerUseRoot(home: string = homedir()): string { + return join(home, '.qwen', 'computer-use'); +} + +/** Directory a given version's assets extract into. */ +export function versionDir( + home: string = homedir(), + version: string = CUA_DRIVER_VERSION, +): string { + return join(computerUseRoot(home), `cua-driver-rs-${version}`); +} + +/** + * Absolute path to the spawnable `cua-driver` binary for this host. + * `bootstrap` ensures it has been downloaded before `client` spawns it. + */ +export function binaryPath( + home: string = homedir(), + platform: NodeJS.Platform = process.platform, + arch: string = process.arch, + version: string = CUA_DRIVER_VERSION, +): string { + const target = resolveAssetTarget(platform, arch, version); + return join( + versionDir(home, version), + target.extractDir, + target.binaryRelPath, ); } + +/** + * Stable identity recorded in install-state for first-use approval. + * Bumping the pinned version produces a new key, forcing re-approval + + * re-download of the new binary. + */ +export function approvalKey(version: string = CUA_DRIVER_VERSION): string { + return `cua-driver-rs@${version}`; +} + +function trimSlash(s: string): string { + return s.endsWith('/') ? s.slice(0, -1) : s; +} diff --git a/packages/core/src/tools/computer-use/downloader.smoke.test.ts b/packages/core/src/tools/computer-use/downloader.smoke.test.ts new file mode 100644 index 00000000000..2cb69263bcd --- /dev/null +++ b/packages/core/src/tools/computer-use/downloader.smoke.test.ts @@ -0,0 +1,50 @@ +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, rmSync, existsSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ensureInstalled } from './downloader.js'; +import { binaryPath } from './constants.js'; + +/** + * REAL network + REAL OS extraction smoke. Off by default — only the CI + * "Computer Use download smoke" job sets CUA_DOWNLOAD_SMOKE=1, on each of + * windows-latest / ubuntu-latest / macos-latest. It exercises the actual + * pipeline the unit tests stub out: + * + * fetch checksums.txt + asset (OSS mirror → GitHub fallback) + * → sha256 verify + * → extract (.tar.gz via `tar`; .zip via bsdtar/PowerShell on Windows) + * → resolve the per-OS binary path + * + * This is what actually verifies the Linux (bare-binary tarball, root binary) + * and Windows (.zip, wrapper dir, cua-driver.exe) paths that cannot be run on + * the macOS dev box. Downloads ~20MB, so the timeout is generous. + * + * Because the file matches `*.test.ts`, the normal `test:ci` run collects it + * too — but `describe.runIf` skips the whole block unless the env flag is set, + * so the unit run never hits the network. + */ +const enabled = process.env['CUA_DOWNLOAD_SMOKE'] === '1'; + +describe.runIf(enabled)( + 'cua-driver download smoke (real network + OS extraction)', + () => { + it('downloads, verifies, and extracts the pinned binary for this OS', async () => { + const home = mkdtempSync(join(tmpdir(), 'qwen-cu-smoke-')); + try { + const bin = await ensureInstalled({ home }); + expect(bin).toBe(binaryPath(home)); + expect(existsSync(bin)).toBe(true); + expect(statSync(bin).size).toBeGreaterThan(0); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }, 180_000); + }, +); diff --git a/packages/core/src/tools/computer-use/downloader.test.ts b/packages/core/src/tools/computer-use/downloader.test.ts new file mode 100644 index 00000000000..2f984b095ea --- /dev/null +++ b/packages/core/src/tools/computer-use/downloader.test.ts @@ -0,0 +1,161 @@ +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { + mkdtempSync, + rmSync, + mkdirSync, + writeFileSync, + existsSync, +} from 'node:fs'; +import { createHash } from 'node:crypto'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { + parseChecksums, + findInstalled, + ensureInstalled, +} from './downloader.js'; +import { binaryPath, CUA_DRIVER_VERSION } from './constants.js'; + +describe('parseChecksums', () => { + it('parses sha256sum lines into a filename -> hash map', () => { + const body = [ + 'a'.repeat(64) + ' cua-driver-rs-0.5.2-darwin-arm64.tar.gz', + 'b'.repeat(64) + ' cua-driver-rs-0.5.2-linux-x86_64.tar.gz', + '# a comment line', + '', + ].join('\n'); + const map = parseChecksums(body); + expect(map.get('cua-driver-rs-0.5.2-darwin-arm64.tar.gz')).toBe( + 'a'.repeat(64), + ); + expect(map.size).toBe(2); + }); + + it('tolerates the binary-mode asterisk and stray whitespace', () => { + const map = parseChecksums(`${'c'.repeat(64)} *file.zip\n`); + expect(map.get('file.zip')).toBe('c'.repeat(64)); + }); +}); + +describe('findInstalled / ensureInstalled short-circuit', () => { + let home: string; + + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'qwen-cu-dl-')); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + }); + + it('findInstalled returns undefined when the binary is absent', async () => { + expect(await findInstalled(home, 'darwin', 'arm64')).toBeUndefined(); + }); + + it('findInstalled returns the path once the binary exists', async () => { + const p = binaryPath(home, 'darwin', 'arm64'); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, '#!/bin/sh\n'); + expect(await findInstalled(home, 'darwin', 'arm64')).toBe(p); + }); + + it('ensureInstalled is a no-op (no download) when already installed', async () => { + const p = binaryPath(home, 'darwin', 'arm64'); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, '#!/bin/sh\n'); + + // fetchImpl throws — proving ensureInstalled never reaches the network. + const throwingFetch = (() => { + throw new Error('fetch must not be called when already installed'); + }) as unknown as typeof fetch; + + const result = await ensureInstalled({ + home, + platform: 'darwin', + arch: 'arm64', + version: CUA_DRIVER_VERSION, + fetchImpl: throwingFetch, + }); + expect(result).toBe(p); + }); +}); + +describe('ensureInstalled on Windows (.zip extraction)', () => { + let home: string; + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'qwen-cu-win-')); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + }); + + it('downloads + verifies + unzips, resolving cua-driver.exe under the wrapper dir', async () => { + const asset = `cua-driver-rs-${CUA_DRIVER_VERSION}-windows-x86_64.zip`; + const zipBytes = Buffer.from('PK fake-zip-payload'); + const sha = createHash('sha256').update(zipBytes).digest('hex'); + const checksums = `${sha} ${asset}\n`; + + const fetchImpl = (async (url: string | URL) => { + const u = String(url); + if (u.endsWith('checksums.txt')) return new Response(checksums); + if (u.endsWith(asset)) return new Response(zipBytes); + throw new Error(`unexpected url ${u}`); + }) as unknown as typeof fetch; + + // Stand in for OS unzip: lay down the wrapper dir + exe the real zip holds. + let unzipCalled = false; + const unzipImpl = async (_zip: string, dest: string) => { + unzipCalled = true; + const wrapper = join( + dest, + `cua-driver-rs-${CUA_DRIVER_VERSION}-windows-x86_64`, + ); + mkdirSync(wrapper, { recursive: true }); + writeFileSync(join(wrapper, 'cua-driver.exe'), 'MZ'); + }; + + const result = await ensureInstalled({ + home, + platform: 'win32', + arch: 'x64', + version: CUA_DRIVER_VERSION, + fetchImpl, + unzipImpl, + }); + + expect(unzipCalled).toBe(true); + expect(result).toBe(binaryPath(home, 'win32', 'x64')); + expect(existsSync(result)).toBe(true); + }); + + it('rejects a checksum mismatch before unzipping', async () => { + const asset = `cua-driver-rs-${CUA_DRIVER_VERSION}-windows-x86_64.zip`; + const checksums = `${'0'.repeat(64)} ${asset}\n`; // deliberately wrong hash + const fetchImpl = (async (url: string | URL) => { + const u = String(url); + if (u.endsWith('checksums.txt')) return new Response(checksums); + if (u.endsWith(asset)) return new Response(Buffer.from('payload')); + throw new Error(`unexpected url ${u}`); + }) as unknown as typeof fetch; + + let unzipCalled = false; + await expect( + ensureInstalled({ + home, + platform: 'win32', + arch: 'x64', + version: CUA_DRIVER_VERSION, + fetchImpl, + unzipImpl: async () => { + unzipCalled = true; + }, + }), + ).rejects.toThrow(/checksum mismatch/i); + expect(unzipCalled).toBe(false); + }); +}); diff --git a/packages/core/src/tools/computer-use/downloader.ts b/packages/core/src/tools/computer-use/downloader.ts new file mode 100644 index 00000000000..5272763206e --- /dev/null +++ b/packages/core/src/tools/computer-use/downloader.ts @@ -0,0 +1,313 @@ +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Downloads + installs the pinned cua-driver binary into + * `~/.qwen/computer-use/`. + * + * Source order is OSS mirror → GitHub (see constants.resolveAssetUrls); + * the first reachable source wins. The downloaded asset's sha256 is + * verified against the release `checksums.txt` before extraction, so a + * mirror cannot serve a tampered or truncated binary undetected. + * + * The binaries are Developer-ID-signed + Apple-notarized by Cua AI, Inc., + * so on macOS they pass Gatekeeper without us signing anything. + */ + +import { createHash } from 'node:crypto'; +import { createWriteStream } from 'node:fs'; +import { mkdir, rm, stat, chmod, rename } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { pipeline } from 'node:stream/promises'; +import { Readable } from 'node:stream'; +import { x as tarExtract } from 'tar'; +import { + CUA_DRIVER_VERSION, + binaryPath, + resolveAssetTarget, + resolveAssetUrls, + resolveChecksumUrls, + versionDir, +} from './constants.js'; + +export interface InstallOptions { + home: string; + platform?: NodeJS.Platform; + arch?: string; + version?: string; + env?: NodeJS.ProcessEnv; + /** Progress hook for the bootstrap UI ("Downloading… (~Xs)"). */ + onProgress?: (message: string) => void; + /** Injection point for tests; defaults to global fetch. */ + fetchImpl?: typeof fetch; + /** + * Injection point for unzipping Windows `.zip` assets; defaults to OS tools + * (bsdtar, then PowerShell). Tests and non-bsdtar hosts can override it. + */ + unzipImpl?: (zipPath: string, destDir: string) => Promise; +} + +/** + * Parse a release `checksums.txt` body into a `{ filename -> sha256 }` map. + * Each line is `␠␠` (sha256sum format). + */ +export function parseChecksums(body: string): Map { + const map = new Map(); + for (const line of body.split('\n')) { + const m = line.trim().match(/^([0-9a-f]{64})\s+\*?(.+)$/i); + if (m) map.set(m[2].trim(), m[1].toLowerCase()); + } + return map; +} + +/** Returns the installed binary path if already present, else undefined. */ +export async function findInstalled( + home: string, + platform: NodeJS.Platform = process.platform, + arch: string = process.arch, + version: string = CUA_DRIVER_VERSION, +): Promise { + const p = binaryPath(home, platform, arch, version); + try { + const s = await stat(p); + if (s.isFile()) return p; + } catch { + // not installed + } + return undefined; +} + +/** Fetch the first reachable URL from `urls`, returning the Response body bytes. */ +async function fetchFirst( + urls: string[], + fetchImpl: typeof fetch, + onProgress?: (m: string) => void, +): Promise<{ url: string; res: Response }> { + let lastErr: unknown; + for (const url of urls) { + try { + const res = await fetchImpl(url, { redirect: 'follow' }); + if (res.ok && res.body) return { url, res }; + lastErr = new Error(`HTTP ${res.status} for ${url}`); + } catch (err) { + lastErr = err; + onProgress?.(`Source unreachable, trying fallback…`); + } + } + throw new Error( + `Computer Use: all download sources failed. Last error: ${ + lastErr instanceof Error ? lastErr.message : String(lastErr) + }`, + ); +} + +/** + * Ensure the pinned cua-driver binary is installed, downloading + + * verifying + extracting it if necessary. Returns the binary path. + * Idempotent: a no-op (fast stat) when already installed. + */ +export async function ensureInstalled(opts: InstallOptions): Promise { + const platform = opts.platform ?? process.platform; + const arch = opts.arch ?? process.arch; + const version = opts.version ?? CUA_DRIVER_VERSION; + const env = opts.env ?? process.env; + const fetchImpl = opts.fetchImpl ?? fetch; + const onProgress = opts.onProgress; + + const existing = await findInstalled(opts.home, platform, arch, version); + if (existing) return existing; + + const target = resolveAssetTarget(platform, arch, version); + onProgress?.('Downloading Computer Use driver (~20MB, one time)...'); + + // 1. Resolve expected sha256 from checksums.txt (first reachable source). + const { res: sumRes } = await fetchFirst( + resolveChecksumUrls(env, version), + fetchImpl, + ); + const checksums = parseChecksums(await sumRes.text()); + const expectedSha = checksums.get(target.asset); + if (!expectedSha) { + throw new Error( + `Computer Use: ${target.asset} missing from checksums.txt.`, + ); + } + + // 2. Download the asset to a temp file, hashing as we stream. + const { res } = await fetchFirst( + resolveAssetUrls(target.asset, env, version), + fetchImpl, + onProgress, + ); + await mkdir(computerUseTmp(opts.home), { recursive: true }); + // Name the temp file with the asset's real extension (no `.part`): Windows + // unzip tools key off it — Expand-Archive accepts only `.zip`, and bsdtar + // likewise. A stale half-download is overwritten + re-verified next run. + const tmpFile = join(computerUseTmp(opts.home), target.asset); + const hash = createHash('sha256'); + const nodeStream = Readable.fromWeb(res.body as never); + nodeStream.on('data', (chunk: Buffer) => hash.update(chunk)); + await pipeline(nodeStream, createWriteStream(tmpFile)); + + // 3. Verify sha256 before trusting the bytes. + const actualSha = hash.digest('hex'); + if (actualSha !== expectedSha) { + await rm(tmpFile, { force: true }); + throw new Error( + `Computer Use: checksum mismatch for ${target.asset} ` + + `(expected ${expectedSha.slice(0, 12)}…, got ${actualSha.slice(0, 12)}…).`, + ); + } + + // 4. Extract into the version dir, then atomically expose it. macOS/Linux + // ship .tar.gz (node `tar`); Windows ships .zip (OS unzip — see + // extractArchive, which pulls in no new dependency). + const dir = versionDir(opts.home, version); + const stagingDir = `${dir}.staging`; + await rm(stagingDir, { recursive: true, force: true }); + await mkdir(stagingDir, { recursive: true }); + await extractArchive(tmpFile, stagingDir, target.asset, opts.unzipImpl); + await rm(tmpFile, { force: true }); + await rm(dir, { recursive: true, force: true }); + await rename(stagingDir, dir); + + // 5. Make the binary executable (macOS/Linux only; the exec bit is + // meaningless on Windows and `chmod` there is a no-op at best). + const bin = binaryPath(opts.home, platform, arch, version); + if (platform !== 'win32') { + await chmod(bin, 0o755); + } + + // 6. macOS: prepare CuaDriver.app for the TCC auto-relaunch path. + if (platform === 'darwin' && target.hasApp) { + const extractRoot = join(dir, target.extractDir); + const appDir = join(extractRoot, 'CuaDriver.app'); + // Strip quarantine so the notarized app launches without a Gatekeeper + // prompt (best-effort; notarized binaries pass regardless). + await stripQuarantine(extractRoot); + // Register with LaunchServices so cua-driver's `open -a CuaDriver serve` + // relaunch resolves THIS copy under ~/.qwen — that relaunch is what makes + // TCC attribute Accessibility / Screen Recording to com.trycua.driver + // instead of the launching terminal (iTerm/Terminal/VS Code). Without it + // the auto-relaunch can't find our app and falls back to the terminal's + // TCC identity. Best-effort; non-fatal. + await registerLaunchServices(appDir); + } + + onProgress?.('Computer Use driver ready.'); + return bin; +} + +const LSREGISTER = + '/System/Library/Frameworks/CoreServices.framework/Versions/A/' + + 'Frameworks/LaunchServices.framework/Versions/A/Support/lsregister'; + +/** Register a `.app` with LaunchServices so `open -a ` resolves it. */ +async function registerLaunchServices(appPath: string): Promise { + try { + const { execFile } = await import('node:child_process'); + const { promisify } = await import('node:util'); + await promisify(execFile)(LSREGISTER, ['-f', appPath], { timeout: 15_000 }); + } catch { + // Non-fatal: if registration fails the relaunch may resolve a different + // CuaDriver.app or stay in-process. The driver still works; only the TCC + // identity attribution is affected. + } +} + +function computerUseTmp(home: string): string { + // Keep temp downloads off the user's TMPDIR so a half-download never + // collides with another tool; scope under the install root's parent. + return join( + tmpdir(), + 'qwen-computer-use-dl', + Buffer.from(home).toString('hex').slice(0, 8), + ); +} + +/** Best-effort `xattr -dr com.apple.quarantine` so the notarized app launches clean. */ +async function stripQuarantine(path: string): Promise { + try { + const { execFile } = await import('node:child_process'); + const { promisify } = await import('node:util'); + await promisify(execFile)('xattr', ['-dr', 'com.apple.quarantine', path], { + timeout: 10_000, + }); + } catch { + // Notarized binaries pass Gatekeeper regardless; quarantine strip is a + // belt-and-suspenders nicety. Never fatal. + } +} + +/** + * Extract a downloaded asset into `destDir`. macOS/Linux ship `.tar.gz` + * (handled by the `tar` dep); Windows ships `.zip`, which `tar` cannot read — + * there we shell out to OS unzip tools so we add no new dependency. + */ +async function extractArchive( + archivePath: string, + destDir: string, + asset: string, + unzipImpl?: (zipPath: string, destDir: string) => Promise, +): Promise { + if (asset.endsWith('.zip')) { + await (unzipImpl ?? extractZipWindows)(archivePath, destDir); + } else { + await tarExtract({ file: archivePath, cwd: destDir }); + } +} + +/** + * Unzip a `.zip` on Windows with no new dependency, trying OS tools in order: + * 1. bsdtar (`tar.exe`) — bundled since Windows 10 1803 (2018). Reads zip, + * fast, one-shot; most modern hosts take this path. + * 2. PowerShell `Expand-Archive` — fallback for hosts without bsdtar. + * Surfaces both failures if neither is available. + */ +async function extractZipWindows( + zipPath: string, + destDir: string, +): Promise { + const { execFile } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const run = promisify(execFile); + try { + // `--force-local`: Windows bsdtar otherwise parses the `C:\…` archive path + // as a remote `host:path` and fails with "Cannot connect to C: resolve failed". + await run('tar', ['--force-local', '-xf', zipPath, '-C', destDir], { + timeout: 120_000, + }); + } catch (bsdtarErr) { + try { + await run( + 'powershell', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `Expand-Archive -LiteralPath ${psSingleQuote(zipPath)} ` + + `-DestinationPath ${psSingleQuote(destDir)} -Force`, + ], + { timeout: 180_000 }, + ); + } catch (psErr) { + throw new Error( + `Computer Use: failed to unzip ${zipPath} on Windows ` + + `(bsdtar: ${errMsg(bsdtarErr)}; PowerShell: ${errMsg(psErr)}).`, + ); + } + } +} + +/** Quote a string as a PowerShell single-quoted literal (`'` → `''`). */ +function psSingleQuote(s: string): string { + return `'${s.replace(/'/g, "''")}'`; +} + +function errMsg(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} diff --git a/packages/core/src/tools/computer-use/permission-detector.test.ts b/packages/core/src/tools/computer-use/permission-detector.test.ts index f0d7976bee8..35bb495be63 100644 --- a/packages/core/src/tools/computer-use/permission-detector.test.ts +++ b/packages/core/src/tools/computer-use/permission-detector.test.ts @@ -1,3 +1,9 @@ +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + import { describe, it, expect } from 'vitest'; import { detectPermissionError } from './permission-detector.js'; import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; @@ -19,26 +25,38 @@ describe('detectPermissionError', () => { ).toBe('none'); }); - it('detects accessibility permission missing (upstream phrasing)', () => { - // From AccessibilitySnapshot.swift:104 - const result = textErrorResult( - 'Accessibility permission is required. Run `open-computer-use doctor` and grant access to Open Computer Use.', - ); - expect(detectPermissionError(result)).toBe('accessibility'); + it('detects accessibility missing (cua-driver "Accessibility: NOT granted")', () => { + expect( + detectPermissionError(textErrorResult('❌ Accessibility: NOT granted.')), + ).toBe('accessibility'); }); - it('detects screen recording permission missing', () => { - const result = textErrorResult( - 'Screen Recording permission is required to capture this window.', - ); - expect(detectPermissionError(result)).toBe('screenRecording'); + it('detects screen recording missing (cua-driver "Screen Recording: missing")', () => { + expect( + detectPermissionError( + textErrorResult( + '✅ Accessibility: granted.\n❌ Screen Recording: missing.', + ), + ), + ).toBe('screenRecording'); }); - it('detects via the generic doctor marker as fallback', () => { - const result = textErrorResult( - 'Some unfamiliar error. Run `open-computer-use doctor` for help.', - ); - expect(detectPermissionError(result)).toBe('unknown_permission'); + it('detects via the generic "needs your permission" fallback', () => { + expect( + detectPermissionError( + textErrorResult( + 'cua-driver needs your permission before `serve` can start.', + ), + ), + ).toBe('unknown_permission'); + }); + + it('detects via the generic "Missing TCC grant" fallback', () => { + expect( + detectPermissionError( + textErrorResult('Missing TCC grant(s) for this process.'), + ), + ).toBe('unknown_permission'); }); it('returns "other" for unrelated errors', () => { diff --git a/packages/core/src/tools/computer-use/permission-detector.ts b/packages/core/src/tools/computer-use/permission-detector.ts index 0a8056eb472..e588ed5355a 100644 --- a/packages/core/src/tools/computer-use/permission-detector.ts +++ b/packages/core/src/tools/computer-use/permission-detector.ts @@ -7,32 +7,39 @@ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; /** - * What kind of permission issue, if any, the upstream MCP result - * indicates. We classify based on message strings because upstream - * doesn't expose typed error codes through MCP (see - * `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/Errors.swift` - * in the open-codex-computer-use repo). - * - * Long-term fix is to PR upstream for a typed errorKind; for now this - * string detection is the contract. + * What kind of permission issue, if any, the cua-driver MCP result + * indicates. We classify based on message strings because cua-driver + * doesn't expose a typed errorKind through MCP. The strings below are + * taken from cua-driver's macOS permission surface + * (`libs/cua-driver/rust/crates/platform-macos/src/permissions/`). */ export type PermissionErrorKind = | 'none' // success, or non-error result | 'other' // error, but not a permission issue | 'accessibility' // AX missing | 'screenRecording' // Screen Recording missing - | 'unknown_permission'; // matches the doctor marker but doesn't pinpoint which + | 'unknown_permission'; // permission-related but doesn't pinpoint which /** - * Upstream-known error patterns. Order matters — more specific - * patterns first. + * cua-driver permission error patterns. Order matters — more specific + * patterns first; the generic "Missing TCC grant" / "needs your + * permission" fallbacks are last so they don't preempt the specific ones. */ const PATTERNS: Array<{ kind: PermissionErrorKind; regex: RegExp }> = [ - { kind: 'accessibility', regex: /accessibility permission is required/i }, + { + kind: 'accessibility', + regex: /accessibility:?\s*(missing|denied|not granted)/i, + }, + { kind: 'accessibility', regex: /accessibility permission/i }, + { + kind: 'screenRecording', + regex: /screen recording:?\s*(missing|denied|not granted)/i, + }, { kind: 'screenRecording', regex: /screen recording permission/i }, - // Fallback: any error mentioning the doctor command is likely permission-related. - // Listed last so it doesn't preempt the specific patterns. - { kind: 'unknown_permission', regex: /open-computer-use\s+doctor/i }, + { + kind: 'unknown_permission', + regex: /missing tcc grant|needs your permission/i, + }, ]; export function detectPermissionError( diff --git a/packages/core/src/tools/computer-use/registration.test.ts b/packages/core/src/tools/computer-use/registration.test.ts index 4fd134a01c8..b8ec3174497 100644 --- a/packages/core/src/tools/computer-use/registration.test.ts +++ b/packages/core/src/tools/computer-use/registration.test.ts @@ -16,8 +16,8 @@ describe('registerComputerUseTools', () => { await registerComputerUseTools(registerLazy as never); - expect(registerLazy).toHaveBeenCalledTimes(9); - expect(registered).toHaveLength(9); + expect(registerLazy).toHaveBeenCalledTimes(COMPUTER_USE_TOOL_NAMES.length); + expect(registered).toHaveLength(COMPUTER_USE_TOOL_NAMES.length); for (const name of COMPUTER_USE_TOOL_NAMES) { expect(registered).toContain(`computer_use__${name}`); } @@ -37,10 +37,10 @@ describe('registerComputerUseTools', () => { await registerComputerUseTools(registerLazy as never); - // registerLazy IS called for all 9 (the gate runs inside it), but only - // 7 land in `registered` because click + drag were denied. - expect(registerLazy).toHaveBeenCalledTimes(9); - expect(registered).toHaveLength(7); + // registerLazy IS called for every curated tool (the gate runs inside + // it), but click + drag are denied so they don't land in `registered`. + expect(registerLazy).toHaveBeenCalledTimes(COMPUTER_USE_TOOL_NAMES.length); + expect(registered).toHaveLength(COMPUTER_USE_TOOL_NAMES.length - 2); expect(registered).not.toContain('computer_use__click'); expect(registered).not.toContain('computer_use__drag'); }); diff --git a/packages/core/src/tools/computer-use/schemas.test.ts b/packages/core/src/tools/computer-use/schemas.test.ts index 3c2005b14b4..7f9b8be56f8 100644 --- a/packages/core/src/tools/computer-use/schemas.test.ts +++ b/packages/core/src/tools/computer-use/schemas.test.ts @@ -1,14 +1,56 @@ +/** + * @license + * Copyright 2025 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + import { describe, it, expect } from 'vitest'; import { COMPUTER_USE_SCHEMAS, COMPUTER_USE_TOOL_NAMES } from './schemas.js'; -describe('computer-use schemas', () => { - it('exports exactly 9 schemas', () => { - expect(Object.keys(COMPUTER_USE_SCHEMAS)).toHaveLength(9); +describe('computer-use schemas (cua-driver full tool surface)', () => { + it('exports the complete cua-driver tool set (no curation)', () => { + // Every tool cua-driver advertises is exposed; if upstream adds/removes + // tools, re-run scripts/sync-computer-use-schemas.ts and bump this count. + expect(Object.keys(COMPUTER_USE_SCHEMAS)).toHaveLength(35); + expect(COMPUTER_USE_TOOL_NAMES).toHaveLength(35); + }); + + it('includes the renamed screenshot+AX tool (get_window_state, not get_app_state)', () => { + expect(COMPUTER_USE_TOOL_NAMES).toContain('get_window_state'); + expect(COMPUTER_USE_TOOL_NAMES).not.toContain('get_app_state'); + }); + + it('includes the page (CDP/Electron) tool and other full-surface tools', () => { + // `page` reaches Electron/webview content the native AX tree can't — + // it must NOT be curated out. + for (const t of [ + 'page', + 'launch_app', + 'kill_app', + 'start_session', + 'move_cursor', + 'set_config', + 'get_accessibility_tree', + ]) { + expect(COMPUTER_USE_TOOL_NAMES).toContain(t); + } }); - it('each tool name matches the upstream convention (no computer_use__ prefix)', () => { - // schemas.ts uses upstream names verbatim ("click", "type_text"). - // The computer_use__ prefix lives on the qwen-code-facing wrapper. + it('keeps the core action tools', () => { + for (const t of [ + 'list_apps', + 'click', + 'scroll', + 'drag', + 'type_text', + 'press_key', + 'set_value', + ]) { + expect(COMPUTER_USE_TOOL_NAMES).toContain(t); + } + }); + + it('each tool name is an upstream name (no computer_use__ prefix)', () => { for (const name of COMPUTER_USE_TOOL_NAMES) { expect(name).not.toContain('computer_use__'); expect(name).toMatch(/^[a-z_]+$/); @@ -26,23 +68,24 @@ describe('computer-use schemas', () => { } }); - it('list_apps takes no parameters', () => { - expect(COMPUTER_USE_SCHEMAS.list_apps.parameterSchema).toEqual({ - type: 'object', - properties: {}, - additionalProperties: false, - }); + it('list_apps takes no required parameters', () => { + const schema = COMPUTER_USE_SCHEMAS.list_apps.parameterSchema as { + required?: string[]; + properties?: Record; + }; + expect(schema.required ?? []).toHaveLength(0); }); - it('click requires app and either element_index or x/y', () => { + it('click targets a pid (cua-driver semantics, not the old ocu app string)', () => { const schema = COMPUTER_USE_SCHEMAS.click.parameterSchema as { properties: Record; required: string[]; }; - expect(schema.properties).toHaveProperty('app'); + expect(schema.properties).toHaveProperty('pid'); expect(schema.properties).toHaveProperty('element_index'); expect(schema.properties).toHaveProperty('x'); expect(schema.properties).toHaveProperty('y'); - expect(schema.required).toContain('app'); + expect(schema.required).toContain('pid'); + expect(schema.properties).not.toHaveProperty('app'); }); }); diff --git a/packages/core/src/tools/computer-use/schemas.ts b/packages/core/src/tools/computer-use/schemas.ts index d07ab9fd840..859d5c50d1b 100644 --- a/packages/core/src/tools/computer-use/schemas.ts +++ b/packages/core/src/tools/computer-use/schemas.ts @@ -5,20 +5,14 @@ */ /** - * Hardcoded schemas for the @qwen-code/open-computer-use tools. + * Hardcoded schemas for the cua-driver computer-use tools. * - * Pinned to: @qwen-code/open-computer-use@0.2.3 - * (Exact pin — see PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME / - * PINNED_OPEN_COMPUTER_USE_VERSION in constants.ts for the canonical - * package + version and bump procedure.) + * Pinned to: cua-driver-rs v0.5.2 (see CUA_DRIVER_VERSION in constants.ts). * - * Verified against the published package: `tools/list` returns the - * same 9 tools (click, drag, get_app_state, list_apps, - * perform_secondary_action, press_key, scroll, set_value, type_text). - * The 0.2.0 → 0.2.3 changes (notarization, the window-free - * `permission-status` CLI command, the screenshot minScale-clamp fix) do - * not alter the MCP tool surface, so these schemas are unchanged across - * all of them. + * Generated from the live `cua-driver mcp` `tools/list` output — the FULL + * tool surface cua-driver advertises (no curation). Exposing the complete set + * (including page/CDP, cursor, session, recording, config, kill_app, …) so the + * model has every capability cua-driver actually provides. * * Regenerated by scripts/sync-computer-use-schemas.ts — do not hand-edit. */ @@ -29,15 +23,41 @@ export interface ComputerUseToolSchema { } export const COMPUTER_USE_TOOL_NAMES = [ + 'bring_to_front', + 'check_for_update', + 'check_permissions', 'click', + 'double_click', 'drag', - 'get_app_state', + 'end_session', + 'get_accessibility_tree', + 'get_agent_cursor_state', + 'get_config', + 'get_cursor_position', + 'get_recording_state', + 'get_screen_size', + 'get_window_state', + 'hotkey', + 'kill_app', + 'launch_app', 'list_apps', - 'perform_secondary_action', + 'list_windows', + 'move_cursor', + 'page', 'press_key', + 'replay_trajectory', + 'right_click', 'scroll', + 'set_agent_cursor_enabled', + 'set_agent_cursor_motion', + 'set_agent_cursor_style', + 'set_config', 'set_value', + 'start_recording', + 'start_session', + 'stop_recording', 'type_text', + 'zoom', ] as const; export type ComputerUseToolName = (typeof COMPUTER_USE_TOOL_NAMES)[number]; @@ -46,207 +66,1012 @@ export const COMPUTER_USE_SCHEMAS: Record< ComputerUseToolName, ComputerUseToolSchema > = { - click: { + bring_to_front: { + description: + 'Activate a window so subsequent input tools with `dispatch:"foreground"` land on it without a per-call SetForegroundWindow flash. **Windows-only:** on macOS this tool returns an error pointing to the platform-native `NSRunningApplication.activate` (which the macOS input tools don\'t need because CGEvent.postToPid reaches backgrounded windows). On Linux this tool also stubs out; use `wmctrl -a` or `xdotool windowactivate` if you need explicit activation.', + parameterSchema: { + additionalProperties: false, + properties: { + pid: { + type: 'integer', + }, + window_id: { + type: 'integer', + }, + }, + required: ['pid'], + type: 'object', + }, + }, + check_for_update: { description: - 'Click an element by index or pixel coordinates from screenshot. This tool is part of plugin `Computer Use`.', + 'Check whether a newer cua-driver-rs release is available on GitHub. Returns the current and latest versions, an `update_available` boolean, the install one-liner, and the release notes URL. Read-only — never installs. Mirror of `cua-driver check-update --json`.', parameterSchema: { + additionalProperties: false, + properties: {}, type: 'object', + }, + }, + check_permissions: { + description: + "Report TCC permission status for Accessibility and Screen Recording. By default also raises the system permission dialogs for any missing grants — Apple's request APIs are no-ops when the grant is already active, so this is safe to call repeatedly. Pass {\"prompt\": false} for a purely read-only status check.\n\nReturns: `accessibility` + `screen_recording` (booleans from the TCC preflight APIs), `screen_recording_capturable` (a live ScreenCaptureKit probe — if it disagrees with `screen_recording`, the preflight grant belongs to a different process), and `source` (which TCC identity the booleans reflect: the CuaDriver daemon vs the launching terminal/IDE). macOS attributes grants to the responsible process, so a standalone call from a terminal reports the terminal's grants, not the driver's.", + parameterSchema: { + additionalProperties: false, + properties: { + prompt: { + description: + 'Raise the system permission prompts for missing grants. Default true.', + type: 'boolean', + }, + }, + type: 'object', + }, + }, + click: { + description: + "Left-click against a target pid. **Prefer `element_index` over pixel coordinates** — element_index works on backgrounded / minimized / hidden / off-Space windows, surfaces a stable handle that survives rebuilds, and tells you what you're clicking via the cached element's role + label. Reach for `x, y` only when the target is a canvas / video / WebGL / custom-drawn surface that doesn't appear in the AX tree.\n\nTwo addressing modes:\n\n- element_index + window_id (from last get_window_state): AX action path. Works on backgrounded/hidden windows. No cursor move, no focus steal. element_index cache is scoped per (pid, window_id) and is replaced by the next snapshot of the same window — re-snapshot every turn before clicking.\n\n- x, y (window-local screenshot pixels, top-left origin of the PNG returned by get_window_state): CGEvent path. Synthesizes mouse events and posts to pid. Use modifier for cmd/shift/option/ctrl. Needs a visible on-screen window to anchor the conversion.\n\naction: press (default), show_menu, pick, confirm, cancel, open.\nfrom_zoom: set true after a zoom call to auto-translate zoom-image pixel coordinates to full-window space.", + parameterSchema: { + additionalProperties: false, properties: { - click_count: { + action: { + description: + 'AX action: press, show_menu, pick, confirm, cancel, open.', + type: 'string', + }, + count: { + description: 'Click count (pixel path only). Default 1.', type: 'integer', - description: 'Number of clicks. Defaults to 1', }, - mouse_button: { - description: 'Mouse button to click. Defaults to left.', - enum: ['left', 'right', 'middle'], + debug_image_out: { + description: + 'Optional file path. When set on a pixel-addressed click, captures a fresh screenshot, draws a red crosshair at (x, y), and writes the PNG. Use to verify coordinate spaces. Requires window_id; incompatible with from_zoom.', type: 'string', }, element_index: { + description: 'Element index from last get_window_state.', + type: 'integer', + }, + from_zoom: { + description: + 'When true, x and y are in the last zoom image for this pid; driver translates back to full-window coordinates.', + type: 'boolean', + }, + modifier: { + description: 'Modifier keys: cmd, shift, option/alt, ctrl.', + items: { + type: 'string', + }, + type: 'array', + }, + pid: { + description: 'Target process ID.', + type: 'integer', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', type: 'string', - description: 'Element index to click', + }, + window_id: { + description: 'Target window ID. Required for element_index.', + type: 'integer', + }, + x: { + description: 'Window-local screenshot X coordinate.', + type: 'number', }, y: { + description: 'Window-local screenshot Y coordinate.', type: 'number', - description: 'Y coordinate in screenshot pixel coordinates', }, - app: { + }, + required: ['pid'], + type: 'object', + }, + }, + double_click: { + description: + "Double-click at (x, y) or on an AX element identified by element_index + window_id.\n\nAX path (element_index provided): performs `AXOpen` when the element advertises it (Finder items, openable list rows/cells); otherwise resolves the element's on-screen center and falls back to a pixel double-click there.\n\nPixel path (x, y provided): two down/up pairs ~80 ms apart at the given coordinates.", + parameterSchema: { + additionalProperties: false, + properties: { + element_index: { + description: + 'Element index from last get_window_state. Uses AX path.', + type: 'integer', + }, + pid: { + type: 'integer', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', type: 'string', - description: 'App name or bundle identifier', + }, + window_id: { + description: 'CGWindowID. Required when element_index is used.', + type: 'integer', }, x: { - description: 'X coordinate in screenshot pixel coordinates', + description: 'Screen X coordinate (pixel path).', + type: 'number', + }, + y: { + description: 'Screen Y coordinate (pixel path).', type: 'number', }, }, - required: ['app'], - additionalProperties: false, + required: ['pid'], + type: 'object', }, }, drag: { description: - 'Drag from one point to another using pixel coordinates. This tool is part of plugin `Computer Use`.', + "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels — the same space get_window_state returns. Top-left origin of the target's window.\n\nUse for: marquee/lasso selection, drag-and-drop, resizing via a handle, scrubbing a slider, repositioning a panel.\n\n`duration_ms` (default 500) is the wall-clock budget for the path between mouse-down and mouse-up; `steps` (default 20) is the number of intermediate mouseDragged events linearly interpolated along the path. Increase both for slower, more human drags; decrease for snap gestures.\n\n`modifier` keys (cmd/shift/option/ctrl) are held across the entire gesture.\n\nWhen `from_zoom` is true, coordinates are in the last zoom image for this pid; the driver maps them back to window coordinates before dispatching.", parameterSchema: { - type: 'object', + additionalProperties: false, properties: { - app: { + button: { + description: 'Mouse button used for the drag. Default: left.', + enum: ['left', 'right', 'middle'], type: 'string', - description: 'App name or bundle identifier', + }, + duration_ms: { + description: + 'Wall-clock duration of the drag path between mouseDown and mouseUp. Default: 500.', + maximum: 10000, + minimum: 0, + type: 'integer', }, from_x: { - description: 'Start X coordinate', + description: + 'Drag-start X in window-local screenshot pixels. Top-left origin.', type: 'number', }, from_y: { + description: + 'Drag-start Y in window-local screenshot pixels. Top-left origin.', type: 'number', - description: 'Start Y coordinate', + }, + from_zoom: { + description: + 'When true, coordinates are in the last zoom image for this pid; driver maps back to window coordinates.', + type: 'boolean', + }, + modifier: { + description: + 'Modifier keys held across the entire gesture: cmd/shift/option/ctrl.', + items: { + type: 'string', + }, + type: 'array', + }, + pid: { + description: 'Target process ID.', + type: 'integer', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', + type: 'string', + }, + steps: { + description: + 'Number of intermediate mouseDragged events linearly interpolated along the path. Default: 20.', + maximum: 200, + minimum: 1, + type: 'integer', }, to_x: { - description: 'End X coordinate', + description: 'Drag-end X in window-local screenshot pixels.', type: 'number', }, to_y: { + description: 'Drag-end Y in window-local screenshot pixels.', type: 'number', - description: 'End Y coordinate', + }, + window_id: { + description: + 'CGWindowID for the window the pixel coordinates were measured against. Optional — when omitted the driver picks the frontmost window of pid.', + type: 'integer', }, }, - required: ['app', 'from_x', 'from_y', 'to_x', 'to_y'], - additionalProperties: false, + required: ['pid', 'from_x', 'from_y', 'to_x', 'to_y'], + type: 'object', + }, + }, + end_session: { + description: + "End a session declared with `start_session`: removes its agent cursor, stops any recording it owns, and clears its per-session config. Call this when a run finishes so its cursor doesn't linger (otherwise the idle-TTL reclaims it after a period of inactivity). Idempotent.", + parameterSchema: { + additionalProperties: true, + properties: { + session: { + description: 'The session id to end.', + type: 'string', + }, + }, + required: ['session'], + type: 'object', }, }, - get_app_state: { + get_accessibility_tree: { description: - "Start an app use session if needed, then get the state of the app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app. This tool is part of plugin `Computer Use`.", + "Return a lightweight snapshot of the desktop: running regular apps and on-screen visible windows with their bounds, z-order, and owner pid.\n\nFor the full AX subtree of a single window (with interactive element indices you can click by), use `get_window_state` instead — that's the heavy per-window tool. This one is a fast discovery read that needs no TCC grants.", parameterSchema: { + additionalProperties: false, + properties: {}, type: 'object', + }, + }, + get_agent_cursor_state: { + description: + "Return the current state of THIS session's agent cursor: position, config (color, icon, label, size, opacity), enabled flag. Pass cursor_id to inspect a specific instance.", + parameterSchema: { + additionalProperties: false, properties: { - app: { - description: 'App name or bundle identifier', + cursor_id: { + description: "Cursor instance. Default: this session's cursor.", type: 'string', }, }, - required: ['app'], + type: 'object', + }, + }, + get_config: { + description: 'Return the current cua-driver-rs configuration.', + parameterSchema: { additionalProperties: false, + properties: {}, + type: 'object', }, }, - list_apps: { + get_cursor_position: { description: - 'List the apps on this computer. Returns the set of apps that are currently running, as well as any that have been used in the last 14 days, including details on usage frequency. This tool is part of plugin `Computer Use`.', + 'Return the current mouse cursor position in screen points (origin top-left).', parameterSchema: { + additionalProperties: false, + properties: {}, type: 'object', + }, + }, + get_recording_state: { + description: + 'Report the current trajectory recorder state: whether recording is enabled, the output directory (when enabled), and the 1-based counter for the next turn folder that will be written. Counter increments on every recorded action tool call and resets to 1 each time recording is (re-)enabled.\n\nPure read-only.', + parameterSchema: { + additionalProperties: false, properties: {}, + type: 'object', + }, + }, + get_screen_size: { + description: + 'Return the logical size of the main display in points plus its backing scale factor. Agents click in points; Retina displays have scale_factor 2.0. Requires no TCC permissions.', + parameterSchema: { additionalProperties: false, + properties: {}, + type: 'object', }, }, - perform_secondary_action: { + get_window_state: { description: - 'Invoke a secondary accessibility action exposed by an element. This tool is part of plugin `Computer Use`.', + "Walk a running app's AX tree and return a Markdown rendering of its UI, tagging every actionable element with [element_index N]. Pass those indices to click, type_text, press_key, etc.\n\nINVARIANT: call get_window_state once per turn per (pid, window_id) before any element-indexed action. The index map is replaced by the next snapshot.\n\nAlso captures a PNG screenshot of the specified window.\n\nOptional `query` filters the tree_markdown to matching lines plus their ancestor chain (case-insensitive substring). The element_index values are unchanged — filtering only trims the rendered Markdown.", parameterSchema: { + additionalProperties: false, + properties: { + capture_mode: { + description: + 'som=AX+screenshot (default), vision=screenshot only (no AX walk), ax=AX only (no screenshot).', + enum: ['som', 'vision', 'ax'], + type: 'string', + }, + pid: { + description: 'Target process ID.', + type: 'integer', + }, + query: { + description: 'Case-insensitive filter for tree_markdown.', + type: 'string', + }, + screenshot_out_file: { + description: + 'When set, write the PNG to this file path (~ expanded) instead of embedding base64 in the response. The structured output will contain screenshot_file_path instead.', + type: 'string', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', + type: 'string', + }, + window_id: { + description: 'Target window ID from list_windows.', + type: 'integer', + }, + }, + required: ['pid', 'window_id'], type: 'object', + }, + }, + hotkey: { + description: + 'Press a combination of keys simultaneously — e.g. `["cmd", "c"]` for Copy, `["cmd", "shift", "4"]` for screenshot selection. The combo is posted directly to the target pid\'s event queue; the target does NOT need to be frontmost.\n\nTwo delivery paths:\n• Default (no window_id): auth-message envelope — Chromium/Electron apps accept the keystrokes as trusted live input on macOS 14+.\n• With window_id: NSMenu path — briefly activates the target WindowServer-frontmost via SLPSSetFrontProcessWithOptions (kCPSNoWindows, < 1 ms), posts WITHOUT the auth envelope so IOHIDPostEvent fires and NSApplication.sendEvent: dispatches NSMenu key equivalents (e.g. Cmd+Z undo, Cmd+W close). Restores prior frontmost immediately. Use this path when you need native menu-bar actions on non-Chromium apps.\n\nRecognized modifiers: cmd/command, shift, option/alt, ctrl/control, fn. Non-modifier keys use the same vocabulary as `press_key` (return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, letters, digits). Order: modifiers first, one non-modifier last.', + parameterSchema: { + additionalProperties: false, properties: { - action: { - description: 'Secondary accessibility action name', + keys: { + description: + 'Modifier(s) and one non-modifier key, e.g. ["cmd", "c"].', + items: { + type: 'string', + }, + minItems: 2, + type: 'array', + }, + pid: { + description: 'Target process ID.', + type: 'integer', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', type: 'string', }, - app: { - description: 'App name or bundle identifier', + window_id: { + description: + 'When set, uses NSMenu path: briefly activates the window for menu key dispatch, then restores prior frontmost.', + type: 'integer', + }, + }, + required: ['pid', 'keys'], + type: 'object', + }, + }, + kill_app: { + description: + 'Force-terminate a process by pid (kill -9 equivalent on macOS / Linux; taskkill /F equivalent on Windows). Use as escalation when the cooperative close path (hotkey cmd+q on macOS, click-the-X on Windows) failed to make the process exit. Unsaved state is lost — prefer the cooperative path first.', + parameterSchema: { + additionalProperties: false, + properties: { + pid: { + description: 'PID of the process to terminate.', + type: 'integer', + }, + }, + required: ['pid'], + type: 'object', + }, + }, + launch_app: { + description: + 'Launch a macOS app in the background — the target does NOT come to the foreground.\n\nProvide either `bundle_id` (preferred — unambiguous, e.g. `com.apple.calculator`) or `name` (e.g. "Calculator"). If both are given, bundle_id wins.\n\nOptional `urls` are handed to the app as open targets — for Finder, pass a folder path to open a backgrounded Finder window there.\n\nOptional `electron_debugging_port`: opens a Chrome DevTools Protocol (CDP) server on the specified port (appends --remote-debugging-port=N to the app\'s argv). Use this to automate Electron/VS Code/Cursor via CDP.\n\nOptional `webkit_inspector_port`: opens a WebKit inspector server on the specified port (sets WEBKIT_INSPECTOR_SERVER=127.0.0.1:N + TAURI_WEBVIEW_AUTOMATION=1). Use this for Tauri/WebKit-based apps.\n\nOptional `creates_new_application_instance`: when true, forces a new app instance even if one is already running (passes -n to open). Reach for this when another agent or session may drive the SAME app concurrently — it returns a fresh pid + window so each session acts on its own isolated window instead of clobbering one shared instance. Without it, single-instance apps (Calculator, many utilities) hand every caller the same window, so two sessions fight over it.\n\nOptional `additional_arguments`: extra argv strings appended after --args.\n\nReturns the launched app\'s pid, bundle_id, name, and a `windows` array (same shape as `list_windows`) so callers can skip an extra round-trip before `get_window_state(pid, window_id)`. When the focus-steal belt-and-braces demotion check ran (target pid ≠ prior frontmost), the response also includes `self_activation_suppressed: bool` — true if focus stayed with the prior frontmost, false if the launched app held focus despite the re-demote attempt.', + parameterSchema: { + additionalProperties: false, + properties: { + additional_arguments: { + description: 'Extra arguments appended after --args when launching.', + items: { + type: 'string', + }, + type: 'array', + }, + bundle_id: { + description: + 'App bundle identifier, e.g. com.apple.calculator. Preferred over name.', type: 'string', }, - element_index: { - description: 'Element identifier', + creates_new_application_instance: { + description: + 'When true, force a new app instance even if already running (open -n). Use for concurrent multi-agent/multi-session work so each session gets an isolated instance + window instead of sharing one — on single-instance apps (e.g. Calculator) every caller otherwise gets the same window and the sessions clobber each other.', + type: 'boolean', + }, + electron_debugging_port: { + description: + 'Open a Chrome DevTools Protocol server on this port (appends --remote-debugging-port=N).', + type: 'integer', + }, + name: { + description: 'App display name. Used only when bundle_id is absent.', type: 'string', }, + urls: { + description: + 'Optional file paths or URLs to open with the app (e.g. a folder path for Finder).', + items: { + type: 'string', + }, + type: 'array', + }, + webkit_inspector_port: { + description: + 'Open a WebKit inspector server on this port (sets WEBKIT_INSPECTOR_SERVER env var).', + type: 'integer', + }, }, - required: ['app', 'element_index', 'action'], + type: 'object', + }, + }, + list_apps: { + description: + 'List macOS apps — both currently running and installed-but-not-running — with per-app state flags:\n\n- running: is a process for this app live? (pid is 0 when false)\n- active: is it the system-frontmost app? (implies running)\n- launch_path: filesystem path to the `.app` bundle, when known. Pass this to `launch_app` to start the app cold.\n- kind: `"desktop"` for `.app` bundles on macOS.\n- last_used: RFC3339 timestamp from the bundle\'s filesystem mtime, when readable; otherwise null.\n\nOnly apps with NSApplicationActivationPolicyRegular are included — background helpers and system UI agents are filtered out. Installed apps come from scanning /Applications, /Applications/Utilities, ~/Applications, /System/Applications, and /System/Applications/Utilities.\n\nUse this for "is X installed?" as well as "is X running?". For per-window state — on-screen, on-current-Space, minimized, window titles — call list_windows instead. For just opening an app — running or not — call launch_app({bundle_id: ...}) directly; list_apps is not a prerequisite.', + parameterSchema: { additionalProperties: false, + properties: {}, + type: 'object', }, }, - press_key: { + list_windows: { description: - 'Press a key or key-combination on the keyboard, including modifier and navigation keys.\n - This supports xdotool\'s `key` syntax.\n - Examples: "a", "Return", "Tab", "super+c", "Up", "KP_0" (for the numpad 0 key). This tool is part of plugin `Computer Use`.', + 'List all layer-0 top-level windows currently known to WindowServer. Includes off-screen windows (minimized, on another Space, hidden-launched). Use this to find a window_id before calling get_window_state.\n\nPer-record fields: window_id, pid, app_name, title, bounds (x/y/width/height, top-left origin), z_index (higher = frontmost), is_on_screen, on_current_space.', parameterSchema: { + additionalProperties: false, + properties: { + on_screen_only: { + description: + 'When true, drop windows not on the current Space. Default false.', + type: 'boolean', + }, + pid: { + description: + "Optional pid filter. When set, only this pid's windows are returned.", + type: 'integer', + }, + }, type: 'object', + }, + }, + move_cursor: { + description: + "Move the agent cursor overlay to (x, y). Does NOT move the real mouse cursor — the user's cursor stays where it is. Useful for showing the agent's attention without interrupting the user.", + parameterSchema: { + additionalProperties: false, + properties: { + cursor_id: { + description: "Cursor instance to move. Default: 'default'.", + type: 'string', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', + type: 'string', + }, + x: { + type: 'number', + }, + y: { + type: 'number', + }, + }, + required: ['x', 'y'], + type: 'object', + }, + }, + page: { + description: + "Interact with the browser page loaded in a running app. Supports Chrome, Brave, Edge, Safari (via AppleScript on macOS), Electron apps (via CDP), Chromium/Firefox on Windows (via UIA for read; CDP for execute_javascript when --remote-debugging-port is set), and WKWebView/Tauri/AT-SPI fallbacks.\n\nActions:\n- execute_javascript: Run JS and return the result.\n- get_text: Extract visible text from the page.\n- query_dom: Find elements matching a CSS selector.\n- click_element: Click a CSS-selected element AND animate the agent cursor to its on-screen center first (so the user sees what the agent is doing). Prefer over `execute_javascript('el.click()')` whenever you want visible cursor feedback.\n- enable_javascript_apple_events: macOS-only — patch the browser's Preferences to allow JS from Apple Events (Chrome/Brave/Edge, requires user confirmation and a browser restart).", + parameterSchema: { + additionalProperties: false, properties: { - app: { - description: 'App name or bundle identifier', + action: { + description: 'Action to perform.', + enum: [ + 'execute_javascript', + 'get_text', + 'query_dom', + 'click_element', + 'enable_javascript_apple_events', + ], + type: 'string', + }, + attributes: { + description: 'Element attributes to include in query_dom results.', + items: { + type: 'string', + }, + type: 'array', + }, + bundle_id: { + description: + 'Bundle ID of the browser. Required for enable_javascript_apple_events (macOS only).', + type: 'string', + }, + css_selector: { + description: + "CSS selector for query_dom (e.g. 'a', 'button', 'input', 'h1'-'h6', 'p', 'img', 'select', '*').", + type: 'string', + }, + javascript: { + description: + 'JavaScript to execute. Required for execute_javascript.', + type: 'string', + }, + pid: { + description: 'Target process ID.', + type: 'integer', + }, + selector: { + description: + "CSS selector for click_element (e.g. 'button.submit', '#login a').", type: 'string', }, + user_has_confirmed_enabling: { + description: + 'Must be true to proceed with enable_javascript_apple_events. This will quit and relaunch the browser.', + type: 'boolean', + }, + window_id: { + description: 'Target window ID from list_windows.', + type: 'integer', + }, + }, + required: ['action'], + type: 'object', + }, + }, + press_key: { + description: + 'Press and release a single key, delivered to the target pid via CGEventPostToPid. No focus steal.\n\nTwo delivery paths:\n• window_id + element_index: focuses the AX element first, then posts via the auth-message path (Chromium-safe).\n• window_id only (no element_index): NSMenu path — briefly activates the window WindowServer-frontmost via SLPSSetFrontProcessWithOptions (kCPSNoWindows, < 1 ms), posts WITHOUT the auth envelope so IOHIDPostEvent fires and NSApplication.sendEvent: dispatches NSMenu key equivalents. Restores prior frontmost immediately.\n• No window_id: standard auth-message path.\n\nKey names: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, plus any letter or digit.\nModifiers array: cmd, shift, option/alt, ctrl, fn.', + parameterSchema: { + additionalProperties: false, + properties: { + element_index: { + type: 'integer', + }, key: { + description: 'Key name: return, tab, escape, up, down, etc.', + type: 'string', + }, + modifiers: { + description: 'Modifier keys: cmd, shift, option/alt, ctrl, fn.', + items: { + type: 'string', + }, + type: 'array', + }, + pid: { + type: 'integer', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', + type: 'string', + }, + window_id: { + type: 'integer', + }, + }, + required: ['pid', 'key'], + type: 'object', + }, + }, + replay_trajectory: { + description: + "Replay a recorded trajectory by re-invoking every turn's tool call in lexical order. `dir` must point at a directory previously written by `start_recording`. Each `turn-NNNNN/` is parsed for `action.json`, and the recorded tool is called with its recorded `arguments` via the same dispatch path an MCP / CLI call uses.\n\nCaveats:\n- Element-indexed actions (`click({pid, element_index})` etc.) will fail because element indices are per-snapshot and don't survive across sessions. Pixel clicks (`click({pid, x, y})`) and all keyboard tools replay cleanly. Failures are reported but don't stop replay unless `stop_on_error` is true.\n- `get_window_state` and other read-only tools are NOT currently recorded, so replays do not re-populate the per-(pid, window_id) element cache.\n- If recording is ENABLED while replay runs, the replay itself is recorded into the currently configured output directory. That's deliberate: recording a replay against a new build and diffing the two trajectories is the regression-test workflow.", + parameterSchema: { + additionalProperties: false, + properties: { + delay_ms: { + description: + 'Milliseconds to sleep between turns, for human-observable pacing. Default 500.', + maximum: 10000, + minimum: 0, + type: 'integer', + }, + dir: { + description: + 'Trajectory directory previously written by `set_recording`. Absolute or ~-rooted.', type: 'string', - description: 'Key or key combination to press', + }, + stop_on_error: { + description: + 'Stop replay on the first tool-call error. Default true — set false to best-effort through the full trajectory.', + type: 'boolean', }, }, - required: ['app', 'key'], + required: ['dir'], + type: 'object', + }, + }, + right_click: { + description: + "Right-click against a target pid. Two addressing modes:\n\n- `element_index` + `window_id` (from the last `get_window_state` snapshot) — performs `AXShowMenu` on the cached element. Pure AX RPC, works on backgrounded / hidden windows, no cursor move or focus steal. Requires a prior `get_window_state(pid, window_id)` in this turn.\n\n- `x`, `y` — synthesizes `rightMouseDown` / `rightMouseUp` CGEvent pair posted to the pid. Driver converts image-pixel → screen-point internally. `modifier` forces the CGEvent path (AX actions don't propagate modifier keys).\n\nExactly one of `element_index` or (`x` AND `y`) must be provided. `pid` always required. `window_id` required when `element_index` is used.", + parameterSchema: { additionalProperties: false, + properties: { + element_index: { + description: + 'Element index from last get_window_state. Routes through AXShowMenu. Requires window_id.', + type: 'integer', + }, + modifier: { + description: + 'Modifier keys held during the right-click: cmd/shift/option/ctrl. Pixel path only.', + items: { + type: 'string', + }, + type: 'array', + }, + pid: { + description: 'Target process ID.', + type: 'integer', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', + type: 'string', + }, + window_id: { + description: 'CGWindowID. Required when element_index is used.', + type: 'integer', + }, + x: { + description: + 'X in window-local screenshot pixels. Must be provided together with y.', + type: 'number', + }, + y: { + description: + 'Y in window-local screenshot pixels. Must be provided together with x.', + type: 'number', + }, + }, + required: ['pid'], + type: 'object', }, }, scroll: { description: - 'Scroll an element in a direction by a number of pages. This tool is part of plugin `Computer Use`.', + "Scroll the target pid's focused region by synthesized keystrokes.\n\nMapping: by='page' → PageDown/PageUp × amount; by='line' → DownArrow/UpArrow × amount. Horizontal variants use Left/Right arrow keys.\n\nOptional element_index + window_id pre-focuses the element before scrolling.", + parameterSchema: { + additionalProperties: false, + properties: { + amount: { + description: 'Number of keystroke repetitions. Default: 3.', + maximum: 50, + minimum: 1, + type: 'integer', + }, + by: { + description: 'Scroll granularity. Default: line.', + enum: ['line', 'page'], + type: 'string', + }, + direction: { + description: 'Scroll direction.', + enum: ['up', 'down', 'left', 'right'], + type: 'string', + }, + element_index: { + type: 'integer', + }, + pid: { + type: 'integer', + }, + session: { + description: + 'Optional session id: declares/uses the agent cursor and per-session state for this run. The same id works over MCP, the CLI, or the raw socket, and follows the run across apps/windows. Omit to run cursor-less.', + type: 'string', + }, + window_id: { + type: 'integer', + }, + }, + required: ['pid', 'direction'], + type: 'object', + }, + }, + set_agent_cursor_enabled: { + description: + "Show or hide the agent cursor for a session. A cursor exists only for a DECLARED session: pass `session` (the same id you start_session / drive actions with) and the cursor appears on that session's first action — its color is derived from the id. Without a `session`, actions run cursor-less. Use enabled=false to hide a session's cursor, enabled=true to re-show it. (`cursor_id` is a legacy alias for `session`.)", parameterSchema: { + additionalProperties: false, + properties: { + cursor_id: { + description: "Cursor instance. Default: 'default'.", + type: 'string', + }, + enabled: { + description: 'true = show, false = hide.', + type: 'boolean', + }, + }, + required: ['enabled'], type: 'object', + }, + }, + set_agent_cursor_motion: { + description: + "Configure the visual appearance and motion curve of an agent cursor instance.\n\nAppearance (multi-cursor customization):\n- cursor_id: instance name (default='default')\n- cursor_icon: built-in ('arrow','crosshair','hand','dot') or PNG/SVG file path\n- cursor_color: hex color e.g. '#00FFFF' or CSS name\n- cursor_label: short text shown near the cursor\n- cursor_size: dot radius in points (default=16)\n- cursor_opacity: 0.0–1.0 (default=0.85)\n\nMotion curve (Bezier path shape):\n- start_handle: departure control-point fraction [0,1]. Default 0.3\n- end_handle: arrival control-point fraction [0,1]. Default 0.3\n- arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n- arc_flow: asymmetry [-1,1]; positive bulges toward destination. Default 0.0\n- spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n- glide_duration_ms: fixed flight duration per move [50,5000]; omit for speed-based (the default)\n- dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n- idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000", + parameterSchema: { + additionalProperties: false, properties: { - pages: { + arc_flow: { + description: 'Asymmetry bias in [-1, 1]. Default 0.0.', type: 'number', + }, + arc_size: { description: - 'Number of pages to scroll. Fractional values are supported. Defaults to 1', + 'Arc deflection as fraction of path length [0, 1]. Default 0.25.', + type: 'number', }, - app: { - description: 'App name or bundle identifier', + cursor_color: { + description: "Hex color (e.g. '#00FFFF') or CSS color name.", type: 'string', }, - element_index: { - description: 'Element identifier', + cursor_icon: { + description: 'Built-in icon name or file path to PNG/SVG.', type: 'string', }, - direction: { - description: 'Scroll direction: up, down, left, or right', + cursor_id: { + description: "Cursor instance name. Default: 'default'.", + type: 'string', + }, + cursor_label: { + description: 'Short label near the cursor dot.', + type: 'string', + }, + cursor_opacity: { + description: 'Opacity 0.0–1.0. Default: 0.85.', + type: 'number', + }, + cursor_size: { + description: 'Dot radius in points. Default: 16.', + type: 'number', + }, + dwell_after_click_ms: { + description: 'Pause after click ripple in ms. Default 80.', + maximum: 5000, + minimum: 0, + type: 'number', + }, + end_handle: { + description: 'End-handle fraction in [0, 1]. Default 0.3.', + type: 'number', + }, + glide_duration_ms: { + description: + 'Fixed flight duration per move in ms; omit for speed-based timing (the default).', + maximum: 5000, + minimum: 50, + type: 'number', + }, + idle_hide_ms: { + description: 'Auto-hide delay in ms. 0 = never hide. Default 20000.', + maximum: 60000, + minimum: 0, + type: 'number', + }, + spring: { + description: 'Settle damping in [0.3, 1.0]. Default 0.72.', + type: 'number', + }, + start_handle: { + description: 'Start-handle fraction in [0, 1]. Default 0.3.', + type: 'number', + }, + turn_radius: { + description: + 'Minimum turning radius of the glide path in points; smaller = tighter curves. Default 80.', + maximum: 1000, + minimum: 1, + type: 'number', + }, + }, + type: 'object', + }, + }, + set_agent_cursor_style: { + description: + 'Update the visual style of the agent cursor overlay.\n\n- gradient_colors: array of CSS hex strings (e.g. ["#FF0000","#0000FF"]) used as the arrow fill gradient from tip to tail. Empty array reverts to the default palette colours.\n- bloom_color: hex string for the radial halo/bloom behind the cursor (e.g. "#00FFFF"). Empty string reverts to the default.\n- image_path: path to a PNG, JPEG, SVG, or ICO file to use as the cursor icon instead of the default gradient arrow. Empty string reverts to the procedural arrow.\nAll parameters are optional; omit any you do not want to change.', + parameterSchema: { + additionalProperties: false, + properties: { + bloom_color: { + description: + "Hex bloom/halo colour (e.g. '#00FFFF'). '' = revert to default.", + type: 'string', + }, + cursor_id: { + description: "Cursor instance. Default: 'default'.", + type: 'string', + }, + gradient_colors: { + description: + 'CSS hex gradient stops tip→tail. [] = revert to default.', + items: { + type: 'string', + }, + type: 'array', + }, + image_path: { + description: + "Path to PNG/JPEG/SVG/ICO cursor image. '' = revert to arrow.", type: 'string', }, }, - required: ['app', 'element_index', 'direction'], + type: 'object', + }, + }, + set_config: { + description: + 'Update cua-driver-rs configuration. Changes to capture_mode and max_image_dimension take effect immediately. The experimental_pip keys are persisted to ~/.cua-driver/config.json and take effect on the next daemon restart (the PiP backend is initialised once at startup).', + parameterSchema: { additionalProperties: false, + properties: { + capture_mode: { + description: 'Default capture mode for get_window_state.', + enum: ['som', 'vision', 'ax'], + type: 'string', + }, + experimental_pip: { + description: + 'Enable the experimental picture-in-picture preview window. Applies on next daemon restart.', + type: 'boolean', + }, + experimental_pip_geometry: { + description: + 'PiP window size + optional position in `WxH` or `WxH+X+Y` form (e.g. `320x200+24+24`). Applies on next daemon restart.', + type: 'string', + }, + max_image_dimension: { + description: 'Max dimension for screenshot resizing (0 = no limit).', + type: 'integer', + }, + }, + type: 'object', }, }, set_value: { description: - 'Set the value of a settable accessibility element. This tool is part of plugin `Computer Use`.', + 'Set a value on a UI element. Two modes depending on element role:\n\n- **AXPopUpButton / select dropdown**: finds the child option whose title or value matches `value` (case-insensitive) and AXPresses it directly — the native macOS popup menu is never opened, so focus is never stolen. Use this for HTML