Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 17 additions & 15 deletions docs/users/configuration/settings.md

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions packages/cli/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1900,6 +1900,8 @@ export async function loadCliConfig(
cronEnabled: settings.experimental?.cron ?? true,
agentTeamEnabled: settings.experimental?.agentTeam ?? false,
computerUseEnabled: settings.tools?.computerUse?.enabled ?? true,
computerUseMaxImageDimension:
settings.tools?.computerUse?.maxImageDimension,
emitToolUseSummaries: settings.experimental?.emitToolUseSummaries ?? true,
listExtensions: argv.listExtensions || false,
overrideExtensions: overrideExtensions || argv.extensions,
Expand Down
10 changes: 10 additions & 0 deletions packages/cli/src/config/settingsSchema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1993,6 +1993,16 @@ const SETTINGS_SCHEMA = {
'When enabled (default), the cua-driver computer_use__* tools are registered as deferred built-ins.',
showInDialog: true,
},
maxImageDimension: {
type: 'number',
label: 'Max Screenshot Dimension',
category: 'Tools',
requiresRestart: true,
default: -1,
description:
"Longest-edge pixel cap applied to cua-driver screenshots (via set_config's max_image_dimension). -1 (default) keeps cua-driver's built-in default (1568); 0 disables resizing (full resolution); a positive value caps the longest edge. Lower caps cut vision-token cost at the expense of fine detail. Overridable via the QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION env var.",
showInDialog: false,
},
},
},
},
Expand Down
13 changes: 13 additions & 0 deletions packages/core/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -809,6 +809,7 @@ export interface ConfigParameters {
agentTeamEnabled?: boolean;
workflowsEnabled?: boolean;
computerUseEnabled?: boolean;
computerUseMaxImageDimension?: number;
emitToolUseSummaries?: boolean;
listExtensions?: boolean;
overrideExtensions?: string[];
Expand Down Expand Up @@ -1241,6 +1242,7 @@ export class Config {
private readonly agentTeamEnabled: boolean = false;
private workflowsEnabled = false;
private readonly computerUseEnabled: boolean = true;
private readonly computerUseMaxImageDimension?: number;
private readonly emitToolUseSummaries: boolean = true;
private readonly chatRecordingEnabled: boolean;
private readonly loadMemoryFromIncludeDirectories: boolean = false;
Expand Down Expand Up @@ -1434,6 +1436,7 @@ export class Config {
this.agentTeamEnabled = params.agentTeamEnabled ?? false;
this.workflowsEnabled = params.workflowsEnabled ?? false;
this.computerUseEnabled = params.computerUseEnabled ?? true;
this.computerUseMaxImageDimension = params.computerUseMaxImageDimension;
this.emitToolUseSummaries = params.emitToolUseSummaries ?? true;
this.listExtensions = params.listExtensions ?? false;
this.overrideExtensions = params.overrideExtensions;
Expand Down Expand Up @@ -3833,6 +3836,16 @@ export class Config {
return this.computerUseEnabled;
}

/**
* Configured screenshot longest-edge cap for Computer Use, or `undefined`
* to leave cua-driver's built-in default (1568) in place. Resolved together
* with the `QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION` env override at the point
* the driver connects (see `resolveMaxImageDimension`).
*/
getComputerUseMaxImageDimension(): number | undefined {
return this.computerUseMaxImageDimension;
}

/**
* Whether the turn loop should fire a fast-model call after each tool batch
* to emit a `tool_use_summary` message. Mirrors Claude Code's
Expand Down
74 changes: 74 additions & 0 deletions packages/core/src/tools/computer-use/client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,80 @@ describe('ComputerUseClient', () => {
});
});

// ---------------------------------------------------------------------------
// applyRuntimeConfig — set_config(max_image_dimension) on (re)connect.
// Exercised directly (it's a private method) with a fake inner MCP client, so
// it runs without spawning a real cua-driver binary.
// ---------------------------------------------------------------------------

describe('applyRuntimeConfig (set_config on connect)', () => {
type Inner = { callTool: ReturnType<typeof vi.fn> };

const invokeApply = (
c: ComputerUseClient,
inner: Inner,
progress: (m: string) => void,
) =>
(
c as unknown as {
applyRuntimeConfig: (
client: unknown,
progress: (m: string) => void,
) => Promise<void>;
}
).applyRuntimeConfig(inner, progress);

it('pushes max_image_dimension via set_config when an override is configured', async () => {
const inner: Inner = {
callTool: vi.fn().mockResolvedValue({ content: [] }),
};
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
maxImageDimension: 1024,
});
await invokeApply(c, inner, vi.fn());
expect(inner.callTool).toHaveBeenCalledWith({
name: 'set_config',
arguments: { max_image_dimension: 1024 },
});
});

it('applies 0 (disable resizing) as an explicit override, not "unset"', async () => {
const inner: Inner = {
callTool: vi.fn().mockResolvedValue({ content: [] }),
};
const c = new ComputerUseClient({ binary: '/fake/cua-driver' });
c.setMaxImageDimension(0);
await invokeApply(c, inner, vi.fn());
expect(inner.callTool).toHaveBeenCalledWith({
name: 'set_config',
arguments: { max_image_dimension: 0 },
});
});

it('does nothing when no override is set (driver keeps its built-in default)', async () => {
const inner: Inner = { callTool: vi.fn() };
const c = new ComputerUseClient({ binary: '/fake/cua-driver' });
await invokeApply(c, inner, vi.fn());
expect(inner.callTool).not.toHaveBeenCalled();
});

it('never aborts startup when set_config fails — warns via progress and swallows', async () => {
const inner: Inner = {
callTool: vi.fn().mockRejectedValue(new Error('boom')),
};
const progress = vi.fn();
const c = new ComputerUseClient({
binary: '/fake/cua-driver',
maxImageDimension: 800,
});
await expect(invokeApply(c, inner, progress)).resolves.toBeUndefined();
expect(progress).toHaveBeenCalledWith(
expect.stringContaining('max_image_dimension=800'),
);
});
});

// ---------------------------------------------------------------------------
// isTransportClosedError unit tests
// ---------------------------------------------------------------------------
Expand Down
55 changes: 52 additions & 3 deletions packages/core/src/tools/computer-use/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,37 @@ export interface ComputerUseClientOptions {
binary: string;
/** Streaming hook for progress messages during slow operations. */
onProgress?: (message: string) => void;
/**
* Longest-edge pixel cap applied to cua-driver screenshots via `set_config`
* after every (re)connect. `undefined` leaves cua-driver's built-in default
* (1568) untouched; `0` disables resizing. See {@link resolveMaxImageDimension}.
*/
maxImageDimension?: number;
}

export class ComputerUseClient {
private static singleton: ComputerUseClient | undefined;

private readonly binary: string;
private readonly onProgress: (message: string) => void;
private maxImageDimension: number | undefined;
private client: Client | undefined;
private startPromise: Promise<void> | undefined;

constructor(options: ComputerUseClientOptions) {
this.binary = options.binary;
this.onProgress = options.onProgress ?? (() => {});
this.maxImageDimension = options.maxImageDimension;
}

/**
* Set the screenshot longest-edge cap applied on the next (re)connect via
* `set_config`. Cheap to call before every `start()`; the value is only
* pushed to cua-driver inside `doStart` (once per spawn, re-applied after a
* reconnect). `undefined` means "don't override".
*/
setMaxImageDimension(value: number | undefined): void {
this.maxImageDimension = value;
}

/**
Expand Down Expand Up @@ -81,9 +99,9 @@ export class ComputerUseClient {
* and startup messages during this call. It overrides the instance-level
* callback for the duration of the start operation only.
*
* Throws on spawn failure (network down, npx missing, etc.). The
* caller (bootstrap state machine) is responsible for mapping the
* throw into user-facing UX.
* Throws on spawn failure (binary missing / not executable, daemon
* launch failure, etc.). The caller (bootstrap state machine) is
* responsible for mapping the throw into user-facing UX.
*/
async start(onProgress?: (message: string) => void): Promise<void> {
if (this.client) return;
Expand Down Expand Up @@ -111,6 +129,37 @@ export class ComputerUseClient {
);
await client.connect(transport);
this.client = client;
await this.applyRuntimeConfig(client, progress);
}

/**
* Push session-level runtime config to a freshly connected daemon. Today
* that is just `max_image_dimension` (the screenshot longest-edge cap),
* applied via the `set_config` tool when an override is configured.
*
* Runs once per spawn — including after the reconnect in `callTool`, since a
* daemon restart resets runtime config to its persisted default. Best-effort:
* a failed `set_config` must NOT abort startup (the driver is still usable at
* its default dimension), so the error is surfaced via `progress` and
* swallowed. Calls the inner client directly to avoid recursing through
* `callTool`'s reconnect path.
*/
private async applyRuntimeConfig(
client: Client,
progress: (message: string) => void,
): Promise<void> {
if (this.maxImageDimension === undefined) return;
try {
await client.callTool({
name: 'set_config',
arguments: { max_image_dimension: this.maxImageDimension },
});
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
progress(
`Computer Use: could not apply max_image_dimension=${this.maxImageDimension} (${msg}); using driver default.`,
);
}
}

/**
Expand Down
65 changes: 65 additions & 0 deletions packages/core/src/tools/computer-use/constants.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
resolveAssetTarget,
resolveAssetUrls,
resolveChecksumUrls,
resolveMaxImageDimension,
} from './constants.js';

describe('CUA_DRIVER_VERSION', () => {
Expand Down Expand Up @@ -135,3 +136,67 @@ describe('approvalKey', () => {
expect(approvalKey('9.9.9')).toBe('cua-driver-rs@9.9.9');
});
});

describe('resolveMaxImageDimension', () => {
it('returns undefined (no override → cua-driver default) when nothing is set', () => {
expect(resolveMaxImageDimension(undefined, {})).toBeUndefined();
});

it('uses the setting when no env var is present', () => {
expect(resolveMaxImageDimension(1024, {})).toBe(1024);
});

it('treats 0 as an explicit "no resize" override (full resolution)', () => {
expect(resolveMaxImageDimension(0, {})).toBe(0);
expect(
resolveMaxImageDimension(undefined, {
QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION: '0',
}),
).toBe(0);
});

it('treats the -1 sentinel (and any negative) as "use cua-driver default"', () => {
expect(resolveMaxImageDimension(-1, {})).toBeUndefined();
expect(resolveMaxImageDimension(-50, {})).toBeUndefined();
});

it('lets the env var override the setting', () => {
expect(
resolveMaxImageDimension(1024, {
QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION: '768',
}),
).toBe(768);
});

it('falls back to the setting when the env var is invalid (NaN / float / empty / negative)', () => {
for (const bad of ['abc', '12.5', '', ' ', '-1']) {
expect(
resolveMaxImageDimension(1024, {
QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION: bad,
}),
).toBe(1024);
}
});

it('rejects a non-integer / non-finite setting value (no override)', () => {
expect(resolveMaxImageDimension(12.5, {})).toBeUndefined();
expect(resolveMaxImageDimension(Number.NaN, {})).toBeUndefined();
expect(
resolveMaxImageDimension(Number.POSITIVE_INFINITY, {}),
).toBeUndefined();
});

it('reads process.env by default', () => {
const prev = process.env['QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION'];
try {
process.env['QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION'] = '640';
expect(resolveMaxImageDimension(undefined)).toBe(640);
} finally {
if (prev === undefined) {
delete process.env['QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION'];
} else {
process.env['QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION'] = prev;
}
}
});
});
43 changes: 43 additions & 0 deletions packages/core/src/tools/computer-use/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,49 @@ export function resolveChecksumUrls(
return resolveAssetUrls('checksums.txt', env, version);
}

/** Env var name for overriding the screenshot longest-edge cap. */
export const MAX_IMAGE_DIMENSION_ENV = 'QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION';

/**
* Coerce a raw value into a valid `max_image_dimension` override, or
* `undefined` if it isn't one. A valid override is a non-negative integer
* (`0` = no resizing / full resolution). Anything else — negative (incl. the
* `-1` "use default" sentinel), fractional, NaN/Infinity, or empty — yields
* `undefined`, meaning "don't override; let cua-driver use its built-in
* default (1568)".
*/
function coerceImageDimension(
value: string | number | undefined,
): number | undefined {
if (value === undefined) return undefined;
if (typeof value === 'string' && value.trim() === '') return undefined;
const n = typeof value === 'number' ? value : Number(value);
if (!Number.isInteger(n) || n < 0) return undefined;
return n;
}

/**
* Resolve the screenshot longest-edge cap (px) to apply to cua-driver via the
* `set_config` `max_image_dimension` knob. Precedence:
*
* 1. `QWEN_COMPUTER_USE_MAX_IMAGE_DIMENSION` env var (if a valid override)
* 2. the `tools.computerUse.maxImageDimension` setting
* 3. `undefined` → no override; cua-driver keeps its built-in default (1568)
*
* A valid override is a non-negative integer (`0` disables resizing). Negative
* values (incl. the `-1` setting default), non-integers, and blanks mean "no
* override at this layer" — an invalid env value falls through to the setting
* rather than forcing a default.
*/
export function resolveMaxImageDimension(
settingValue?: number,
env: NodeJS.ProcessEnv = process.env,
): number | undefined {
const fromEnv = coerceImageDimension(env[MAX_IMAGE_DIMENSION_ENV]);
if (fromEnv !== undefined) return fromEnv;
return coerceImageDimension(settingValue);
}

/** Install root for all Computer Use artifacts. Footprint stays here. */
export function computerUseRoot(home: string = homedir()): string {
return join(home, '.qwen', 'computer-use');
Expand Down
Loading
Loading