From 2cb54cd96b1caa1575699a4ac2395894d8702e93 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Tue, 14 Jul 2026 04:48:35 +0800 Subject: [PATCH] fix(cu): preserve post-delivery verification state --- .../src/__tests__/cua-driver-backend.test.ts | 114 ++++++++++++++++++ .../computer-use/src/cua-driver-backend.ts | 85 +++++++------ 2 files changed, 164 insertions(+), 35 deletions(-) diff --git a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts index b54a5005e5..c59e72643b 100644 --- a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts +++ b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts @@ -121,6 +121,9 @@ const HANG_ONCE_MARKER = process.env.CUA_MOCK_HANG_ONCE_MARKER || ''; const DELAY_TOOL = process.env.CUA_MOCK_DELAY_TOOL || ''; const DELAY_MS = Number(process.env.CUA_MOCK_DELAY_MS || 0); const ERR_TOOL = process.env.CUA_MOCK_RPCERR_TOOL || ''; +const ERR_AFTER_TOOL = process.env.CUA_MOCK_RPCERR_AFTER_TOOL || ''; +const ERR_AFTER_COUNT = Number(process.env.CUA_MOCK_RPCERR_AFTER_COUNT || 0); +const TOOL_CALL_COUNTS = new Map(); const EMPTY_AX = process.env.CUA_MOCK_EMPTY_AX === '1'; const AX_ROLE = process.env.CUA_MOCK_AX_ROLE || 'AXTextArea'; const AX_LABEL = process.env.CUA_MOCK_AX_LABEL || ''; @@ -164,6 +167,8 @@ function handle(msg) { } if (method === 'tools/call') { const name = params.name; + const toolCallCount = (TOOL_CALL_COUNTS.get(name) || 0) + 1; + TOOL_CALL_COUNTS.set(name, toolCallCount); if (name === HANG_TOOL) { return; } // never respond → exercises abort/kill/handshake-timeout if (name === HANG_ONCE_TOOL && HANG_ONCE_MARKER) { try { @@ -173,6 +178,10 @@ function handle(msg) { } catch (e) {} } if (name === ERR_TOOL) { send({ jsonrpc: '2.0', id: id, error: { code: -32000, message: 'mock rpc error' } }); return; } + if (name === ERR_AFTER_TOOL && toolCallCount >= ERR_AFTER_COUNT) { + send({ jsonrpc: '2.0', id: id, error: { code: -32000, message: 'mock rpc error' } }); + return; + } const sendToolReply = (result) => { if (name === DELAY_TOOL && DELAY_MS > 0) { setTimeout(() => reply(id, result), DELAY_MS); @@ -446,6 +455,8 @@ function makeBackend(opts: { delayTool?: string; delayMs?: number; rpcErrTool?: string; + rpcErrAfterTool?: string; + rpcErrAfterCount?: number; handshakeTimeoutMs?: number; bigImage?: boolean; emptyAx?: boolean; @@ -479,6 +490,8 @@ function makeBackend(opts: { process.env.CUA_MOCK_DELAY_TOOL = opts.delayTool ?? ''; process.env.CUA_MOCK_DELAY_MS = String(opts.delayMs ?? 0); process.env.CUA_MOCK_RPCERR_TOOL = opts.rpcErrTool ?? ''; + process.env.CUA_MOCK_RPCERR_AFTER_TOOL = opts.rpcErrAfterTool ?? ''; + process.env.CUA_MOCK_RPCERR_AFTER_COUNT = String(opts.rpcErrAfterCount ?? 0); process.env.CUA_MOCK_BIG_IMAGE = opts.bigImage ? '1' : ''; process.env.CUA_MOCK_EMPTY_AX = opts.emptyAx ? '1' : ''; process.env.CUA_MOCK_AX_ROLE = opts.axRole ?? 'AXTextArea'; @@ -2308,6 +2321,71 @@ describe('cua-driver backend', () => { assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 1); }); + it('native AX text readback request failure preserves outcome_unknown', async () => { + const { backend, logPath } = makeBackend({ + rpcErrAfterTool: 'get_window_state', + rpcErrAfterCount: 3, + }); + const signal = new AbortController().signal; + const click = await backend.run( + { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction, + signal, + ); + assert.equal(click.outcome.ok, true); + + const result = await backend.run( + { type: 'type', text: 'expected' } as CuAction, + signal, + ); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) { + assert.equal(result.outcome.error, 'outcome_unknown'); + assert.equal(result.outcome.evidence?.path, 'ax'); + } + assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 1); + }); + + it('CDP text inspection request failure preserves outcome_unknown', async () => { + const { backend, logPath } = makeBackend({ + processKind: 'electron', + pageTarget: testPageTarget(), + emptyAx: true, + semanticPointerResult: { + supported: true, + ok: true, + kind: 'left_click', + editable: true, + tagName: 'textarea', + clickEvents: 1, + }, + rpcErrAfterTool: 'page', + rpcErrAfterCount: 5, + }); + const signal = new AbortController().signal; + const click = await backend.run( + { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction, + signal, + ); + assert.equal(click.outcome.ok, true); + + const result = await backend.run( + { type: 'type', text: 'expected' } as CuAction, + signal, + ); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) { + assert.equal(result.outcome.error, 'outcome_unknown'); + assert.equal(result.outcome.evidence?.path, 'cdp'); + } + assert.equal( + businessPageCalls(await readRecords(logPath)) + .filter((call) => call.action === 'insert_text').length, + 1, + ); + }); + it('semantic dispatch followed by oversized fresh capture returns outcome_unknown', async () => { let compressions = 0; const { backend, logPath } = makeBackend({ @@ -2353,6 +2431,42 @@ describe('cua-driver backend', () => { assert.equal(toolCalls(await readRecords(logPath), 'click').length, 1); }); + it('semantic dispatch followed by fresh observation request failure returns outcome_unknown', async () => { + const { backend, logPath } = makeBackend({ + axRole: 'AXButton', + rpcErrAfterTool: 'get_window_state', + rpcErrAfterCount: 3, + }); + const signal = new AbortController().signal; + const context = { + sessionId: 'verification-request-failure', + turnId: 'turn-1', + toolCallId: 'observe', + }; + const observed = await backend.observeApp!({ + app: 'Fixture Window', + includeScreenshot: true, + }, signal, context); + + const result = await backend.runSemantic!({ + type: 'click_element', + observationId: observed.observationId, + elementId: '7', + elementIdentity: observed.elements[0]!.identity, + }, signal, { + ...context, + toolCallId: 'click', + boundAction: boundElementAction(observed, '7'), + }); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) { + assert.equal(result.outcome.error, 'outcome_unknown'); + assert.equal(result.outcome.evidence?.path, 'ax'); + } + assert.equal(toolCalls(await readRecords(logPath), 'click').length, 1); + }); + it('failed screenshot normalization does not evict a usable observation', async () => { let compressions = 0; const { backend } = makeBackend({ diff --git a/packages/computer-use/src/cua-driver-backend.ts b/packages/computer-use/src/cua-driver-backend.ts index 519a57a6b8..4f63176daf 100644 --- a/packages/computer-use/src/cua-driver-backend.ts +++ b/packages/computer-use/src/cua-driver-backend.ts @@ -15,10 +15,11 @@ // // KEYBOARD IS TARGET-BOUND, VERIFIED, AND NEVER FRONTMOST. A successful left // click establishes ownership only for the same Maka session + turn. `type` is -// allowed only for a native, AX-addressable empty field: Maka writes AXValue and -// confirms the value in a fresh snapshot. Electron/unknown processes, non-empty -// fields, and every `key` action fail before any key event is posted. Scroll, -// drag, failed clicks, another session, and another turn never establish ownership. +// allowed for a native AX-addressable empty field, or for a uniquely targeted +// Electron page field through CDP Input.insertText. Both paths require fresh +// readback; unknown processes, non-empty fields, and every `key` action fail +// before key delivery. Scroll, drag, failed clicks, another session, and +// another turn never establish ownership. import { execFile } from 'node:child_process'; import { access } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -427,6 +428,21 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc return { base64, mimeType, widthPx, heightPx }; } + function deliveredVerificationFailure( + actionType: string, + path: 'ax' | 'cdp', + ): CuRunResult { + return { + outcome: { + ok: false, + error: 'outcome_unknown', + message: + `${actionType} was delivered but post-dispatch verification failed`, + evidence: { path, effect: 'unverifiable' }, + }, + }; + } + function clearLocalSession(sessionId: string): void { targetsBySession.delete(sessionId); for (const id of observationIdsBySession.get(sessionId) ?? []) { @@ -1543,7 +1559,15 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc signal, ); if (setResult?.isError) return normalizeCuaDriverOutcome(setResult); - const after = await snapshotTarget(target.window, signal); + let after: TargetSnapshot; + try { + after = await snapshotTarget(target.window, signal); + } catch { + return deliveredVerificationFailure( + 'AXValue write', + 'ax', + ).outcome; + } const verified = editableElementAtScreenPoint( after.elements, target.window.screenPoint, @@ -1650,8 +1674,23 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc signal, ); if (result?.isError) return normalizeCuaDriverOutcome(result); - const inspected = await executePageScript(CUA_INSPECT_PREPARED_ELEMENT_SCRIPT); - if (inspected.response?.isError) return normalizeCuaDriverOutcome(inspected.response); + let inspected: Awaited>; + try { + inspected = await executePageScript( + CUA_INSPECT_PREPARED_ELEMENT_SCRIPT, + ); + } catch { + return deliveredVerificationFailure( + 'CDP Input.insertText', + 'cdp', + ).outcome; + } + if (inspected.response?.isError) { + return deliveredVerificationFailure( + 'CDP Input.insertText', + 'cdp', + ).outcome; + } const after = inspected.element; return after?.editable === true && after.value === text ? { @@ -1916,20 +1955,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc let fresh: CuObservation; try { fresh = await observeResolvedWindow(validated, true, signal, context); - } catch (error) { - if (error instanceof CuaDriverCaptureError) { - return { - outcome: { - ok: false, - error: 'outcome_unknown', - message: - `press_key was delivered but the fresh observation failed: ` - + error.result.outcome.message, - evidence: { path: 'ax', effect: 'unverifiable' }, - }, - }; - } - throw error; + } catch { + return deliveredVerificationFailure('press_key', 'ax'); } return { outcome, @@ -1989,20 +2016,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc signal, context, ); - } catch (error) { - if (error instanceof CuaDriverCaptureError) { - return { - outcome: { - ok: false, - error: 'outcome_unknown', - message: - `${action.type} was delivered but the fresh observation failed: ` - + error.result.outcome.message, - evidence: { path: 'ax', effect: 'unverifiable' }, - }, - }; - } - throw error; + } catch { + return deliveredVerificationFailure(action.type, 'ax'); } return { outcome,