Skip to content

fix(cu): preserve post-delivery verification state #929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Astro-Han merged 1 commit into from
Jul 14, 2026
Merged

fix(cu): preserve post-delivery verification state #929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 114 additions & 0 deletions packages/computer-use/src/__tests__/cua-driver-backend.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ const HANG_ONCE_MARKER = process.env.CUA_MOCK_HANG_ONCE_MARKER || '';
const DELAY_TOOL = process.env.CUA_MOCK_DELAY_TOOL || '';
const DELAY_MS = Number(process.env.CUA_MOCK_DELAY_MS || 0);
const ERR_TOOL = process.env.CUA_MOCK_RPCERR_TOOL || '';
const ERR_AFTER_TOOL = process.env.CUA_MOCK_RPCERR_AFTER_TOOL || '';
const ERR_AFTER_COUNT = Number(process.env.CUA_MOCK_RPCERR_AFTER_COUNT || 0);
const TOOL_CALL_COUNTS = new Map();
const EMPTY_AX = process.env.CUA_MOCK_EMPTY_AX === '1';
const AX_ROLE = process.env.CUA_MOCK_AX_ROLE || 'AXTextArea';
const AX_LABEL = process.env.CUA_MOCK_AX_LABEL || '';
Expand Down Expand Up @@ -164,6 +167,8 @@ function handle(msg) {
}
if (method === 'tools/call') {
const name = params.name;
const toolCallCount = (TOOL_CALL_COUNTS.get(name) || 0) + 1;
TOOL_CALL_COUNTS.set(name, toolCallCount);
if (name === HANG_TOOL) { return; } // never respond → exercises abort/kill/handshake-timeout
if (name === HANG_ONCE_TOOL && HANG_ONCE_MARKER) {
try {
Expand All @@ -173,6 +178,10 @@ function handle(msg) {
} catch (e) {}
}
if (name === ERR_TOOL) { send({ jsonrpc: '2.0', id: id, error: { code: -32000, message: 'mock rpc error' } }); return; }
if (name === ERR_AFTER_TOOL && toolCallCount >= ERR_AFTER_COUNT) {
send({ jsonrpc: '2.0', id: id, error: { code: -32000, message: 'mock rpc error' } });
return;
}
const sendToolReply = (result) => {
if (name === DELAY_TOOL && DELAY_MS > 0) {
setTimeout(() => reply(id, result), DELAY_MS);
Expand Down Expand Up @@ -446,6 +455,8 @@ function makeBackend(opts: {
delayTool?: string;
delayMs?: number;
rpcErrTool?: string;
rpcErrAfterTool?: string;
rpcErrAfterCount?: number;
handshakeTimeoutMs?: number;
bigImage?: boolean;
emptyAx?: boolean;
Expand Down Expand Up @@ -479,6 +490,8 @@ function makeBackend(opts: {
process.env.CUA_MOCK_DELAY_TOOL = opts.delayTool ?? '';
process.env.CUA_MOCK_DELAY_MS = String(opts.delayMs ?? 0);
process.env.CUA_MOCK_RPCERR_TOOL = opts.rpcErrTool ?? '';
process.env.CUA_MOCK_RPCERR_AFTER_TOOL = opts.rpcErrAfterTool ?? '';
process.env.CUA_MOCK_RPCERR_AFTER_COUNT = String(opts.rpcErrAfterCount ?? 0);
process.env.CUA_MOCK_BIG_IMAGE = opts.bigImage ? '1' : '';
process.env.CUA_MOCK_EMPTY_AX = opts.emptyAx ? '1' : '';
process.env.CUA_MOCK_AX_ROLE = opts.axRole ?? 'AXTextArea';
Expand Down Expand Up @@ -2308,6 +2321,71 @@ describe('cua-driver backend', () => {
assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 1);
});

it('native AX text readback request failure preserves outcome_unknown', async () => {
const { backend, logPath } = makeBackend({
rpcErrAfterTool: 'get_window_state',
rpcErrAfterCount: 3,
});
const signal = new AbortController().signal;
const click = await backend.run(
{ type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction,
signal,
);
assert.equal(click.outcome.ok, true);

const result = await backend.run(
{ type: 'type', text: 'expected' } as CuAction,
signal,
);

assert.equal(result.outcome.ok, false);
if (!result.outcome.ok) {
assert.equal(result.outcome.error, 'outcome_unknown');
assert.equal(result.outcome.evidence?.path, 'ax');
}
assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 1);
});

it('CDP text inspection request failure preserves outcome_unknown', async () => {
const { backend, logPath } = makeBackend({
processKind: 'electron',
pageTarget: testPageTarget(),
emptyAx: true,
semanticPointerResult: {
supported: true,
ok: true,
kind: 'left_click',
editable: true,
tagName: 'textarea',
clickEvents: 1,
},
rpcErrAfterTool: 'page',
rpcErrAfterCount: 5,
});
const signal = new AbortController().signal;
const click = await backend.run(
{ type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction,
signal,
);
assert.equal(click.outcome.ok, true);

const result = await backend.run(
{ type: 'type', text: 'expected' } as CuAction,
signal,
);

assert.equal(result.outcome.ok, false);
if (!result.outcome.ok) {
assert.equal(result.outcome.error, 'outcome_unknown');
assert.equal(result.outcome.evidence?.path, 'cdp');
}
assert.equal(
businessPageCalls(await readRecords(logPath))
.filter((call) => call.action === 'insert_text').length,
1,
);
});

it('semantic dispatch followed by oversized fresh capture returns outcome_unknown', async () => {
let compressions = 0;
const { backend, logPath } = makeBackend({
Expand Down Expand Up @@ -2353,6 +2431,42 @@ describe('cua-driver backend', () => {
assert.equal(toolCalls(await readRecords(logPath), 'click').length, 1);
});

it('semantic dispatch followed by fresh observation request failure returns outcome_unknown', async () => {
const { backend, logPath } = makeBackend({
axRole: 'AXButton',
rpcErrAfterTool: 'get_window_state',
rpcErrAfterCount: 3,
});
const signal = new AbortController().signal;
const context = {
sessionId: 'verification-request-failure',
turnId: 'turn-1',
toolCallId: 'observe',
};
const observed = await backend.observeApp!({
app: 'Fixture Window',
includeScreenshot: true,
}, signal, context);

const result = await backend.runSemantic!({
type: 'click_element',
observationId: observed.observationId,
elementId: '7',
elementIdentity: observed.elements[0]!.identity,
}, signal, {
...context,
toolCallId: 'click',
boundAction: boundElementAction(observed, '7'),
});

assert.equal(result.outcome.ok, false);
if (!result.outcome.ok) {
assert.equal(result.outcome.error, 'outcome_unknown');
assert.equal(result.outcome.evidence?.path, 'ax');
}
assert.equal(toolCalls(await readRecords(logPath), 'click').length, 1);
});

it('failed screenshot normalization does not evict a usable observation', async () => {
let compressions = 0;
const { backend } = makeBackend({
Expand Down
85 changes: 50 additions & 35 deletions packages/computer-use/src/cua-driver-backend.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@
//
// KEYBOARD IS TARGET-BOUND, VERIFIED, AND NEVER FRONTMOST. A successful left
// click establishes ownership only for the same Maka session + turn. `type` is
// allowed only for a native, AX-addressable empty field: Maka writes AXValue and
// confirms the value in a fresh snapshot. Electron/unknown processes, non-empty
// fields, and every `key` action fail before any key event is posted. Scroll,
// drag, failed clicks, another session, and another turn never establish ownership.
// allowed for a native AX-addressable empty field, or for a uniquely targeted
// Electron page field through CDP Input.insertText. Both paths require fresh
// readback; unknown processes, non-empty fields, and every `key` action fail
// before key delivery. Scroll, drag, failed clicks, another session, and
// another turn never establish ownership.
import { execFile } from 'node:child_process';
import { access } from 'node:fs/promises';
import { tmpdir } from 'node:os';
Expand Down Expand Up @@ -427,6 +428,21 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc
return { base64, mimeType, widthPx, heightPx };
}

function deliveredVerificationFailure(
actionType: string,
path: 'ax' | 'cdp',
): CuRunResult {
return {
outcome: {
ok: false,
error: 'outcome_unknown',
message:
`${actionType} was delivered but post-dispatch verification failed`,
evidence: { path, effect: 'unverifiable' },
},
};
}

function clearLocalSession(sessionId: string): void {
targetsBySession.delete(sessionId);
for (const id of observationIdsBySession.get(sessionId) ?? []) {
Expand Down Expand Up @@ -1543,7 +1559,15 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc
signal,
);
if (setResult?.isError) return normalizeCuaDriverOutcome(setResult);
const after = await snapshotTarget(target.window, signal);
let after: TargetSnapshot;
try {
after = await snapshotTarget(target.window, signal);
} catch {
return deliveredVerificationFailure(
'AXValue write',
'ax',
).outcome;
}
const verified = editableElementAtScreenPoint(
after.elements,
target.window.screenPoint,
Expand Down Expand Up @@ -1650,8 +1674,23 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc
signal,
);
if (result?.isError) return normalizeCuaDriverOutcome(result);
const inspected = await executePageScript(CUA_INSPECT_PREPARED_ELEMENT_SCRIPT);
if (inspected.response?.isError) return normalizeCuaDriverOutcome(inspected.response);
let inspected: Awaited<ReturnType<typeof executePageScript>>;
try {
inspected = await executePageScript(
CUA_INSPECT_PREPARED_ELEMENT_SCRIPT,
);
} catch {
return deliveredVerificationFailure(
'CDP Input.insertText',
'cdp',
).outcome;
}
if (inspected.response?.isError) {
return deliveredVerificationFailure(
'CDP Input.insertText',
'cdp',
).outcome;
}
const after = inspected.element;
return after?.editable === true && after.value === text
? {
Expand Down Expand Up @@ -1916,20 +1955,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc
let fresh: CuObservation;
try {
fresh = await observeResolvedWindow(validated, true, signal, context);
} catch (error) {
if (error instanceof CuaDriverCaptureError) {
return {
outcome: {
ok: false,
error: 'outcome_unknown',
message:
`press_key was delivered but the fresh observation failed: `
+ error.result.outcome.message,
evidence: { path: 'ax', effect: 'unverifiable' },
},
};
}
throw error;
} catch {
return deliveredVerificationFailure('press_key', 'ax');
}
return {
outcome,
Expand Down Expand Up @@ -1989,20 +2016,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc
signal,
context,
);
} catch (error) {
if (error instanceof CuaDriverCaptureError) {
return {
outcome: {
ok: false,
error: 'outcome_unknown',
message:
`${action.type} was delivered but the fresh observation failed: `
+ error.result.outcome.message,
evidence: { path: 'ax', effect: 'unverifiable' },
},
};
}
throw error;
} catch {
return deliveredVerificationFailure(action.type, 'ax');
}
return {
outcome,
Expand Down
Loading