Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 40 additions & 27 deletions packages/core/src/tools/computer-use/bootstrap.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,11 @@ describe('runBootstrap', () => {
tmpHome = mkdtempSync(join(tmpdir(), 'qwen-cu-bs-'));
deps = {
homeDir: tmpHome,
packageSpec: 'open-computer-use@^0.3.0',
packageSpec: '@qwen-code/open-computer-use@^0.3.0',
platform: 'darwin',
promptInstallApproval: vi.fn(async () => true),
probePermissions: vi.fn(async () => 'ok' as const),
probePermissionStatus: vi.fn(async () => 'ok' as const),
};
});

Expand All @@ -48,7 +49,7 @@ describe('runBootstrap', () => {
it('starts the client when binary is approved + permissions ok', async () => {
const { saveInstallState } = await import('./install-state.js');
await saveInstallState(tmpHome, {
approvedPackageSpec: 'open-computer-use@^0.3.0',
approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0',
approvedAtIso: '2026-05-28T10:00:00Z',
});

Expand Down Expand Up @@ -99,20 +100,27 @@ describe('runBootstrap', () => {

const { loadInstallState } = await import('./install-state.js');
const state = await loadInstallState(tmpHome);
expect(state?.approvedPackageSpec).toBe('open-computer-use@^0.3.0');
expect(state?.approvedPackageSpec).toBe(
'@qwen-code/open-computer-use@^0.3.0',
);
});

it('polls probePermissions when permissions are missing then granted', async () => {
it('shows the window once via doctor, then polls the window-free permission-status until granted', async () => {
const { saveInstallState } = await import('./install-state.js');
await saveInstallState(tmpHome, {
approvedPackageSpec: 'open-computer-use@^0.3.0',
approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0',
approvedAtIso: '2026-05-28T10:00:00Z',
});

let probeCount = 0;
deps.probePermissions = vi.fn(async () => {
probeCount++;
return probeCount < 3 ? 'accessibility' : 'ok';
// Initial probe (doctor) reports missing → enters the poll loop and
// launches the onboarding window ONCE.
deps.probePermissions = vi.fn(async () => 'accessibility' as const);
// Poll probe (permission-status, window-free) reports missing twice
// then granted.
let pollCount = 0;
deps.probePermissionStatus = vi.fn(async () => {
pollCount++;
return pollCount < 2 ? 'accessibility' : 'ok';
});
deps.pollIntervalMs = 1; // speed up test
deps.pollTimeoutMs = 1000;
Expand All @@ -124,23 +132,26 @@ describe('runBootstrap', () => {
deps,
);

// probe is called by bootstrap (each call is a doctor invocation in
// production, but here it's a mock). Doctor itself launches the
// onboarding window when needed — no separate spawnDoctor step.
expect(probeCount).toBeGreaterThanOrEqual(3);
expect(deps.probePermissions).toHaveBeenCalledWith(
'open-computer-use@^0.3.0',
// doctor (window) called exactly once; the window-free status command
// is what gets polled — never doctor again (no window storm).
expect(deps.probePermissions).toHaveBeenCalledTimes(1);
expect(pollCount).toBeGreaterThanOrEqual(2);
expect(deps.probePermissionStatus).toHaveBeenCalledWith(
'@qwen-code/open-computer-use@^0.3.0',
);
});

it('throws after pollTimeoutMs when permissions never grant', async () => {
const { saveInstallState } = await import('./install-state.js');
await saveInstallState(tmpHome, {
approvedPackageSpec: 'open-computer-use@^0.3.0',
approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0',
approvedAtIso: '2026-05-28T10:00:00Z',
});

// Initial doctor reports missing (enters loop); window-free poll never
// grants → must time out.
deps.probePermissions = vi.fn(async () => 'accessibility' as const);
deps.probePermissionStatus = vi.fn(async () => 'accessibility' as const);
deps.pollIntervalMs = 1;
deps.pollTimeoutMs = 50;

Expand All @@ -157,7 +168,7 @@ describe('runBootstrap', () => {
it('skips permission flow on non-darwin platforms', async () => {
const { saveInstallState } = await import('./install-state.js');
await saveInstallState(tmpHome, {
approvedPackageSpec: 'open-computer-use@^0.3.0',
approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0',
approvedAtIso: '2026-05-28T10:00:00Z',
});
deps.platform = 'linux';
Expand All @@ -170,21 +181,23 @@ describe('runBootstrap', () => {
);

expect(deps.probePermissions).not.toHaveBeenCalled();
expect(deps.probePermissionStatus).not.toHaveBeenCalled();
});

it('emits a fresh updateOutput message when permission kind changes mid-poll', async () => {
const { saveInstallState } = await import('./install-state.js');
await saveInstallState(tmpHome, {
approvedPackageSpec: 'open-computer-use@^0.3.0',
approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0',
approvedAtIso: '2026-05-28T10:00:00Z',
});

// Probe sequence: accessibility → screenRecording → ok
let probeCount = 0;
deps.probePermissions = vi.fn(async () => {
probeCount++;
if (probeCount === 1) return 'accessibility' as const;
if (probeCount === 2) return 'screenRecording' as const;
// Initial doctor reports accessibility missing (enters loop, window once).
deps.probePermissions = vi.fn(async () => 'accessibility' as const);
// Window-free poll sequence: accessibility → screenRecording → ok
let pollCount = 0;
deps.probePermissionStatus = vi.fn(async () => {
pollCount++;
if (pollCount === 1) return 'screenRecording' as const;
return 'ok' as const;
});
deps.pollIntervalMs = 1;
Expand All @@ -202,8 +215,7 @@ describe('runBootstrap', () => {
);

// The transition (accessibility → screenRecording) must emit a
// user-facing message naming the new permission kind. LaunchServices
// dedups doctor's window so we don't need a separate spawn step.
// user-facing message naming the new permission kind.
expect(messages.some((m) => m.includes('screenRecording'))).toBe(true);
expect(messages.some((m) => m.includes('accessibility'))).toBe(true);
});
Expand All @@ -215,7 +227,7 @@ describe('runBootstrap', () => {
// probe fire only on a fresh client start.
const { saveInstallState } = await import('./install-state.js');
await saveInstallState(tmpHome, {
approvedPackageSpec: 'open-computer-use@^0.3.0',
approvedPackageSpec: '@qwen-code/open-computer-use@^0.3.0',
approvedAtIso: '2026-05-28T10:00:00Z',
});

Expand All @@ -235,6 +247,7 @@ describe('runBootstrap', () => {

expect(startSpy).not.toHaveBeenCalled();
expect(deps.probePermissions).not.toHaveBeenCalled();
expect(deps.probePermissionStatus).not.toHaveBeenCalled();
});
});

Expand Down
92 changes: 71 additions & 21 deletions packages/core/src/tools/computer-use/bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,22 @@
* On first invocation of any computer_use__* tool:
* 1. If not yet approved: prompt the user to install (one-time).
* 2. Start the client (lazy npx spawn, may take ~60s first time).
* 3. On macOS only: probe permissions via the upstream `doctor` CLI
* (NOT via get_app_state, which has the side-effect of activating
* the target app — earlier rounds probed Finder this way and
* caused Finder to pop to the foreground at session start). The
* doctor command:
* - reads TCC + runtime preflight, prints
* "Permissions: accessibility=granted, screenRecording=missing"
* to stdout, then exits cleanly
* - launches the onboarding window via LaunchServices when any
* permission is missing — LaunchServices dedups so repeated
* invocations just bring the existing window to front
* We parse stdout for the probe result and rely on doctor's own
* window launching for the UX trigger — no separate spawnDoctor
* call needed.
* 3. On macOS only: probe permissions via the upstream CLI (NOT via
* get_app_state, which has the side-effect of activating the target
* app — earlier rounds probed Finder this way and caused Finder to
* pop to the foreground at session start). Two distinct commands:
* - `doctor` (initial probe, once): reads TCC + runtime preflight,
* prints "Permissions: accessibility=..., screenRecording=..."
* to stdout, AND launches the onboarding window when any
* permission is missing. This is what shows the window — once.
* - `permission-status` (poll probe): prints the same summary but
* NEVER launches a window. The poll loop uses this so it does
* not spawn a new onboarding window on every iteration.
* `doctor` does NOT dedup its window — each invocation launches a
* fresh one — so polling `doctor` flooded the screen with windows.
* Probing the window-free `permission-status` in the loop fixes that
* while keeping the "grant then auto-continue" UX.
* Requires @qwen-code/open-computer-use >= 0.2.2 for permission-status.
*/

import { execFile } from 'node:child_process';
Expand Down Expand Up @@ -60,11 +62,20 @@ export interface BootstrapDeps {
*/
promptInstallApproval: (packageSpec: string) => Promise<boolean>;
/**
* Probe permissions by running the upstream doctor CLI and parsing
* its stdout summary. The probe itself triggers the onboarding window
* when permissions are missing — no separate spawnDoctor needed.
* Initial probe: runs `doctor`, which both reports status AND launches
* the onboarding window when permissions are missing. Called exactly
* once on a fresh client start so the onboarding window appears one time.
*/
probePermissions: (packageSpec: string) => Promise<PermissionProbeResult>;
/**
* Poll probe: runs `permission-status`, which reports status WITHOUT
* launching the onboarding window. Called on every poll iteration while
* waiting for the user to grant permissions — using the window-free
* command here is what prevents the onboarding-window storm.
*/
probePermissionStatus: (
packageSpec: string,
) => Promise<PermissionProbeResult>;
/** Poll interval for the permission watcher. Default 5000ms. */
pollIntervalMs?: number;
/** Total poll timeout. Default 10 min. */
Expand Down Expand Up @@ -134,6 +145,41 @@ export async function probePermissionsViaDoctor(
}
}

/**
* Probe macOS permissions via the `permission-status` CLI command —
* the window-free counterpart to `doctor`.
*
* `permission-status` prints the SAME summary line as `doctor` but NEVER
* launches the onboarding window. This is the probe the polling loop uses
* while waiting for the user to grant permissions: `doctor` re-launches a
* fresh onboarding window on every invocation (it does not dedup), so
* polling it every few seconds floods the screen with windows. We launch
* the window exactly once (the initial `doctor` probe) and then poll this
* window-free command.
*
* Requires `@qwen-code/open-computer-use@>=0.2.2`. On older pinned packages
* the command is unknown → npx exits non-zero → we return 'other', which
* the poll loop treats as non-blocking (exits the wait; the real tool call
* then surfaces any permission error). No window storm either way.
*/
export async function probePermissionStatusViaCLI(
packageSpec: string,
): Promise<PermissionProbeResult> {
try {
const { stdout } = await execFileAsync(
'npx',
['-y', packageSpec, 'permission-status'],
{
timeout: 30000,
env: process.env as NodeJS.ProcessEnv,
},
);
return parseDoctorStdout(stdout);
} catch {
return 'other';
}
}

/** Production defaults — instantiated lazily so tests can override per call. */
function defaultDeps(): BootstrapDeps {
const packageSpec = resolveComputerUsePackageSpec();
Expand All @@ -153,6 +199,7 @@ function defaultDeps(): BootstrapDeps {
return process.env['QWEN_COMPUTER_USE_AUTO_APPROVE'] === '1';
},
probePermissions: probePermissionsViaDoctor,
probePermissionStatus: probePermissionStatusViaCLI,
};
}

Expand Down Expand Up @@ -224,9 +271,11 @@ export async function runBootstrap(
);

// Track the last probe kind so we can emit a fresh message on
// transition (e.g. accessibility → screenRecording). LaunchServices
// dedup ensures each subsequent doctor poll re-focuses the existing
// window — no separate spawnDoctor call needed.
// transition (e.g. accessibility → screenRecording). The onboarding
// window was launched once by the initial `doctor` probe above; the
// poll loop below uses the window-free `permission-status` command so
// it never spawns additional windows while the single onboarding
// window stays open.
let lastProbeKind: PermissionProbeResult = probe;

const startedAt = Date.now();
Expand All @@ -240,7 +289,8 @@ export async function runBootstrap(
);
}
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
const next = await deps.probePermissions(deps.packageSpec);
// Window-free status check — see probePermissionStatusViaCLI.
const next = await deps.probePermissionStatus(deps.packageSpec);
if (next === 'ok' || next === 'other') return;

if (next !== lastProbeKind) {
Expand Down
6 changes: 3 additions & 3 deletions packages/core/src/tools/computer-use/client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
describe('ComputerUseClient', () => {
it('is constructible', () => {
const client = new ComputerUseClient({
packageSpec: 'open-computer-use@latest',
packageSpec: '@qwen-code/open-computer-use@latest',
onProgress: vi.fn(),
});
expect(client).toBeDefined();
});

it('reports not-started before start() is called', () => {
const client = new ComputerUseClient({
packageSpec: 'open-computer-use@latest',
packageSpec: '@qwen-code/open-computer-use@latest',
onProgress: vi.fn(),
});
expect(client.isStarted()).toBe(false);
Expand Down Expand Up @@ -129,7 +129,7 @@ class ReconnectTestClient extends ComputerUseClient {

function makeClient(): ReconnectTestClient {
const c = new ReconnectTestClient({
packageSpec: 'open-computer-use@latest',
packageSpec: '@qwen-code/open-computer-use@latest',
});
// Pre-seed the started state so callTool guard passes.
(c as unknown as { client: object }).client = { __fake: true };
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/tools/computer-use/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { resolveComputerUsePackageSpec } from './constants.js';
* action.
*/
export interface ComputerUseClientOptions {
/** npm package spec to npx. Example: "open-computer-use@^0.3.0". */
/** npm package spec to npx. Example: "@qwen-code/open-computer-use@0.2.3". */
packageSpec: string;
/** Streaming hook for progress messages during slow operations. */
onProgress?: (message: string) => void;
Expand Down
20 changes: 16 additions & 4 deletions packages/core/src/tools/computer-use/constants.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import {
PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME,
PINNED_OPEN_COMPUTER_USE_VERSION,
resolveComputerUsePackageSpec,
} from './constants.js';
Expand Down Expand Up @@ -38,16 +39,27 @@ describe('computer-use constants', () => {
});
});

describe('PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME', () => {
it('is the scoped QwenLM fork package', () => {
expect(PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME).toBe(
'@qwen-code/open-computer-use',
);
});
});

describe('resolveComputerUsePackageSpec', () => {
it('defaults to open-computer-use@<PINNED_VERSION> when env var is unset', () => {
it('defaults to <PACKAGE_NAME>@<PINNED_VERSION> when env var is unset', () => {
expect(resolveComputerUsePackageSpec()).toBe(
`open-computer-use@${PINNED_OPEN_COMPUTER_USE_VERSION}`,
`${PINNED_OPEN_COMPUTER_USE_PACKAGE_NAME}@${PINNED_OPEN_COMPUTER_USE_VERSION}`,
);
});

it('honors QWEN_COMPUTER_USE_PACKAGE override', () => {
process.env['QWEN_COMPUTER_USE_PACKAGE'] = 'open-computer-use@0.99.99';
expect(resolveComputerUsePackageSpec()).toBe('open-computer-use@0.99.99');
process.env['QWEN_COMPUTER_USE_PACKAGE'] =
'@qwen-code/open-computer-use@0.99.99';
expect(resolveComputerUsePackageSpec()).toBe(
'@qwen-code/open-computer-use@0.99.99',
);
});

it('reads env var at call time (not at module load)', () => {
Expand Down
Loading
Loading