diff --git a/docs/computer-use-executor-hardening.md b/docs/computer-use-executor-hardening.md index ec0eb8e419..71870642b1 100644 --- a/docs/computer-use-executor-hardening.md +++ b/docs/computer-use-executor-hardening.md @@ -4,6 +4,54 @@ This note records the post-merge review of PR #893 against the current `cua-driver` executor and the local Codex Computer Use reverse-engineering evidence. +## Target Identity Consolidation + +The follow-up review of PRs #930-#933 exposed four variants of the same root +cause: target identity was inferred from mutable content or local indexes +instead of being carried through observation, dispatch, and readback as one +driver-owned identity. + +Evidence: + +- coordinate freshness used one window-wide fingerprint, so removing dynamic + labels and values also removed the identity of the actionable control at the + source coordinate; +- Electron page validation covered only the semantic left/right/double-click + branch, leaving middle-click, triple-click, scroll, and unknown-process pixel + fallback outside the boundary; +- native keyboard ownership came from the pre-click coordinate snapshot and + re-resolved by role/label/value, which could select a different AX node; +- Electron text used a document-global incrementing token installed through a + different helper path from readback, so reload and session reuse were not + fenced. + +The consolidated implementation: + +- keeps the window fingerprint structural, while Runtime binds the smallest + actionable source element's `elementToken`, `elementIndex`, role, label, + value, and frame into the action fingerprint; +- validates Electron page identity before every compatibility pointer fallback + and refuses pixel fallback when process classification is `unknown`; +- refreshes an actionable coordinate against `get_window_state`, requires one + exact role/label/value/frame match, and dispatches AX click with that fresh + element's opaque token; token absence or ambiguity fails closed without pixel + fallback; +- installs and reads the Electron element helper through one bootstrap, with + tokens leased to Maka session, session generation, document fingerprint, and + navigation generation; reload invalidates the lease before insertion; +- sends the fresh `element_token` for semantic `click_element` and `set_value`, + requires `set_value` structured `changed`, `verified`, and `readback_value`, + and carries the readback value into the returned fresh observation; +- keeps native type unsupported until the driver can return the actual focused + element token. The executor does not synthesize a resolver from mutable AX + attributes. + +Remaining driver dependency: + +- pinned `cua-driver` must emit opaque `elements[].element_token` values from + `get_window_state` and accept them on AX click and `set_value`. Native type + remains unavailable until a real focused-token surface exists. + ## Fixed In This Follow-Up - Semantic refetch now requires one unique role/label/value candidate and then diff --git a/packages/computer-use/src/__tests__/computer-use-cumulative-e2e.test.ts b/packages/computer-use/src/__tests__/computer-use-cumulative-e2e.test.ts index 780da4a3c2..af2311b9ae 100644 --- a/packages/computer-use/src/__tests__/computer-use-cumulative-e2e.test.ts +++ b/packages/computer-use/src/__tests__/computer-use-cumulative-e2e.test.ts @@ -39,7 +39,8 @@ function fixtureObservation(overrides: Partial = {}): CuObservati role: 'AXButton', label: 'Commit target', identity: { - token: 'target-button-token', + elementToken: 'target-button-token', + elementIndex: 5, role: 'AXButton', label: 'Commit target', }, diff --git a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts index c59e72643b..bd8c2f650f 100644 --- a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts +++ b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts @@ -80,6 +80,7 @@ function boundCoordinateAction(input: { coordinate?: { x: number; y: number }; zIndex?: number; page?: ComputerUsePageIdentity; + sourceElement?: CuaBoundAction['sourceElement']; } = {}): CuaBoundAction { const pid = input.pid ?? 4242; const windowId = input.windowId ?? 77; @@ -106,6 +107,13 @@ function boundCoordinateAction(input: { sourceCoordinate: coordinate, windowCoordinate: coordinate, coordinateSpace: 'window-screenshot-local', + sourceElement: input.sourceElement ?? { + elementToken: 'snapshot:7', + elementIndex: 7, + role: 'AXTextArea', + value: '', + frame: { x: 250, y: 150, width: 200, height: 120 }, + }, }; } @@ -131,6 +139,7 @@ const SEMANTIC_OCCLUDED = process.env.CUA_MOCK_SEMANTIC_OCCLUDED === '1'; const PAGE_EXEC_RESULT = process.env.CUA_MOCK_PAGE_EXEC_RESULT || ''; const PAGE_READBACK_VALUE = process.env.CUA_MOCK_PAGE_READBACK_VALUE || ''; const NATIVE_READBACK_VALUE = process.env.CUA_MOCK_NATIVE_READBACK_VALUE || ''; +const OMIT_ELEMENT_TOKEN = process.env.CUA_MOCK_OMIT_ELEMENT_TOKEN === '1'; const PAGE_DOCUMENT_MARKER = process.env.CUA_MOCK_PAGE_DOCUMENT_MARKER || 'document-a'; let PAGE_FIELD_VALUE = process.env.CUA_MOCK_PAGE_FIELD_VALUE || ''; let PAGE_INSERTED = false; @@ -210,7 +219,7 @@ function handle(msg) { : { x: 250, y: 150, w: 200, h: 120 }; const baseElement = { element_index: 7, - element_token: 'snapshot:7', + ...(OMIT_ELEMENT_TOKEN ? {} : { element_token: 'snapshot:7' }), role: AX_ROLE, label: AX_LABEL || undefined, value: FIELD_VALUES.has(snapshotWindowId) @@ -312,11 +321,20 @@ function handle(msg) { reply(id, { content: [], structuredContent: { apps: [{ pid: 4242, frontmost: false }] } }); return; case 'set_value': + const requestedValue = String(params.arguments?.value ?? ''); FIELD_VALUES.set( Number(params.arguments?.window_id), - String(params.arguments?.value ?? ''), + requestedValue, ); - reply(id, { content: [{ type: 'text', text: 'value set' }], structuredContent: {} }); + reply(id, { + content: [{ type: 'text', text: 'value set' }], + structuredContent: { + path: 'ax', + changed: true, + verified: true, + readback_value: NATIVE_READBACK_VALUE || requestedValue, + }, + }); return; case 'select_text': case 'perform_secondary_action': @@ -332,7 +350,9 @@ function handle(msg) { const pageText = pageAction === 'execute_javascript' ? pageJavascript.includes('performance.timeOrigin') ? PAGE_DOCUMENT_MARKER - : pageJavascript.includes('__makaComputerUseReadElement') + : pageJavascript.includes('const actionType =') + ? PAGE_EXEC_RESULT + : pageJavascript.includes('__makaComputerUseReadElement') ? JSON.stringify({ editable: true, value: PAGE_INSERTED && PAGE_READBACK_VALUE @@ -468,6 +488,7 @@ function makeBackend(opts: { pageFieldValue?: string; pageReadbackValue?: string; nativeReadbackValue?: string; + omitElementToken?: boolean; pageDocumentMarker?: string; resolvePageDocumentFingerprint?: CuaDriverBackendOptions['resolvePageDocumentFingerprint']; resolveContentFingerprint?: CuaDriverBackendOptions['resolveContentFingerprint']; @@ -503,6 +524,7 @@ function makeBackend(opts: { process.env.CUA_MOCK_PAGE_FIELD_VALUE = opts.pageFieldValue ?? ''; process.env.CUA_MOCK_PAGE_READBACK_VALUE = opts.pageReadbackValue ?? ''; process.env.CUA_MOCK_NATIVE_READBACK_VALUE = opts.nativeReadbackValue ?? ''; + process.env.CUA_MOCK_OMIT_ELEMENT_TOKEN = opts.omitElementToken ? '1' : ''; process.env.CUA_MOCK_PAGE_DOCUMENT_MARKER = opts.pageDocumentMarker ?? 'document-a'; process.env.CUA_MOCK_SNAPSHOT_DELAY_MS = String(opts.snapshotDelayMs ?? 0); process.env.CUA_MOCK_REFETCH_MODE = opts.refetchMode ?? ''; @@ -680,7 +702,8 @@ describe('cua-driver backend', () => { value: '', frame: { x: 250, y: 150, width: 200, height: 120 }, identity: { - token: 'snapshot:7', + elementToken: 'snapshot:7', + elementIndex: 7, role: 'AXButton', value: '', }, @@ -751,7 +774,7 @@ describe('cua-driver backend', () => { assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); }); - it('refetches a unique labeled element when the ephemeral token changes', async () => { + it('dispatches the observed opaque token without attribute-based replacement', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', axLabel: 'Continue', @@ -772,11 +795,11 @@ describe('cua-driver backend', () => { assert.equal(result.outcome.ok, true); const click = toolCall(await readRecords(logPath), 'click'); - assert.equal(click?.element_index, 9); - assert.equal(click?.element_token, 'snapshot:9'); + assert.equal(click?.element_index, 7); + assert.equal(click?.element_token, 'snapshot:7'); }); - it('does not treat a reused token as cross-snapshot semantic identity', async () => { + it('does not resnapshot before semantic dispatch', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', axLabel: 'Continue', @@ -795,41 +818,17 @@ describe('cua-driver backend', () => { elementIdentity: observation.elements[0]!.identity, }, signal, { ...context, boundAction: boundElementAction(observation, '7') }); - assert.equal(result.outcome.ok, false); - if (!result.outcome.ok) assert.match(result.outcome.message, /missing/); - assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); - }); - - it('refetches a tokenless element by one unique role and label match', async () => { - const { backend, logPath } = makeBackend({ - axRole: 'AXButton', - axLabel: 'Continue', - refetchMode: 'replacement', - }); - const signal = new AbortController().signal; - const context = { sessionId: 's1', turnId: 't1', toolCallId: 'semantic' }; - const observation = await backend.observeApp!({ - app: 'Fixture Window', - includeScreenshot: true, - }, signal, context); - const result = await backend.runSemantic!({ - type: 'click_element', - observationId: observation.observationId, - elementId: '7', - elementIdentity: { role: 'AXButton', label: 'Continue' }, - }, signal, { ...context, boundAction: boundElementAction(observation, '7') }); - assert.equal(result.outcome.ok, true); - const click = toolCall(await readRecords(logPath), 'click'); - assert.equal(click?.element_index, 9); - assert.equal(click?.element_token, 'snapshot:9'); + const records = await readRecords(logPath); + assert.equal(toolCalls(records, 'get_window_state').length, 2); + assert.equal(toolCalls(records, 'click')[0]?.element_token, 'snapshot:7'); }); - it('rejects a same-label replacement that moved before semantic dispatch', async () => { + it('fails closed when the observed semantic element has no token', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', axLabel: 'Continue', - refetchMode: 'moved', + omitElementToken: true, }); const signal = new AbortController().signal; const context = { sessionId: 's1', turnId: 't1', toolCallId: 'semantic' }; @@ -845,14 +844,14 @@ describe('cua-driver backend', () => { }, signal, { ...context, boundAction: boundElementAction(observation, '7') }); assert.equal(result.outcome.ok, false); - if (!result.outcome.ok) assert.equal(result.outcome.error, 'stale_frame'); + if (!result.outcome.ok) assert.match(result.outcome.message, /observed semantic element is missing an AX element token/); assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); }); - it('refetches an unlabeled element by unique structural identity', async () => { + it('rejects caller identity that does not match the consumed observation', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', - refetchMode: 'replacement', + axLabel: 'Continue', }); const signal = new AbortController().signal; const context = { sessionId: 's1', turnId: 't1', toolCallId: 'semantic' }; @@ -864,39 +863,19 @@ describe('cua-driver backend', () => { type: 'click_element', observationId: observation.observationId, elementId: '7', - elementIdentity: observation.elements[0]!.identity, + elementIdentity: { + elementToken: 'forged-token', + elementIndex: 7, + role: 'AXButton', + label: 'Continue', + }, }, signal, { ...context, boundAction: boundElementAction(observation, '7') }); - assert.equal(result.outcome.ok, true); - assert.equal(toolCalls(await readRecords(logPath), 'click').length, 1); + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'stale_frame'); + assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); }); - for (const refetchMode of ['missing', 'ambiguous'] as const) { - it(`rejects a ${refetchMode} refetched element without dispatch`, async () => { - const { backend, logPath } = makeBackend({ - axRole: 'AXButton', - axLabel: 'Continue', - refetchMode, - }); - const signal = new AbortController().signal; - const context = { sessionId: 's1', turnId: 't1', toolCallId: 'semantic' }; - const observation = await backend.observeApp!({ - app: 'Fixture Window', - includeScreenshot: true, - }, signal, context); - const result = await backend.runSemantic!({ - type: 'click_element', - observationId: observation.observationId, - elementId: '7', - elementIdentity: observation.elements[0]!.identity, - }, signal, { ...context, boundAction: boundElementAction(observation, '7') }); - - assert.equal(result.outcome.ok, false); - if (!result.outcome.ok) assert.equal(result.outcome.error, 'stale_frame'); - assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); - }); - } - it('declares app observations in capture-local window screenshot space', async () => { let desktopResolverCalls = 0; const { backend } = makeBackend({ @@ -1174,6 +1153,50 @@ describe('cua-driver backend', () => { ); }); + it('rejects a coordinate action when the source actionable element changes meaning', async () => { + const { backend, logPath } = makeBackend({ + axRole: 'AXButton', + axLabel: 'Confirm purchase', + }); + const result = await backend.run( + { type: 'left_click', coordinate: { x: 400, y: 200 } }, + new AbortController().signal, + { + ...DEFAULT_RUN_CONTEXT, + boundAction: boundCoordinateAction({ + sourceElement: { + elementToken: 'snapshot:7', + elementIndex: 7, + role: 'AXButton', + label: 'Delete', + value: '', + frame: { x: 250, y: 150, width: 200, height: 120 }, + }, + }), + }, + ); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'target_changed'); + assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); + }); + + it('fails closed when a fresh actionable coordinate has no element token', async () => { + const { backend, logPath } = makeBackend({ omitElementToken: true }); + const result = await backend.run( + { type: 'left_click', coordinate: { x: 400, y: 200 } }, + new AbortController().signal, + { + ...DEFAULT_RUN_CONTEXT, + boundAction: boundCoordinateAction(), + }, + ); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'target_changed'); + assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); + }); + it('does not treat omitted no-screenshot dimensions as a layout change', async () => { const { backend, logPath } = makeBackend({ emptyAx: true }); const result = await backend.run( @@ -1191,14 +1214,22 @@ describe('cua-driver backend', () => { assert.ok(toolCalls(await readRecords(logPath), 'get_window_state').length > 0); }); - it('keeps an explicit coordinate click on the pixel path over an AX element', async () => { + it('routes an actionable coordinate click through the fresh AX token', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton' }); const result = await backend.run( { type: 'left_click', coordinate: { x: 400, y: 200 } }, new AbortController().signal, { ...DEFAULT_RUN_CONTEXT, - boundAction: boundCoordinateAction(), + boundAction: boundCoordinateAction({ + sourceElement: { + elementToken: 'snapshot:7', + elementIndex: 7, + role: 'AXButton', + value: '', + frame: { x: 250, y: 150, width: 200, height: 120 }, + }, + }), }, ); @@ -1206,10 +1237,10 @@ describe('cua-driver backend', () => { const click = toolCall(await readRecords(logPath), 'click'); assert.equal(click?.pid, 4242); assert.equal(click?.window_id, 77); - assert.equal(typeof click?.x, 'number'); - assert.equal(typeof click?.y, 'number'); - assert.equal(click?.element_index, undefined); - assert.equal(click?.element_token, undefined); + assert.equal(click?.x, undefined); + assert.equal(click?.y, undefined); + assert.equal(click?.element_index, 7); + assert.equal(click?.element_token, 'snapshot:7'); }); it('rejects a bound coordinate occluded by a higher z-order window', async () => { @@ -1348,6 +1379,54 @@ describe('cua-driver backend', () => { assert.equal(toolCalls(records, 'click').length, 0); }); + for (const actionType of ['middle_click', 'triple_click', 'scroll'] as const) { + it(`validates bound Electron page identity before ${actionType} pixel fallback`, async () => { + const { backend, logPath } = makeBackend({ + processKind: 'electron', + pageTarget: testPageTarget(), + }); + const action: CuAction = actionType === 'scroll' + ? { + type: 'scroll', + coordinate: { x: 400, y: 200 }, + scrollDirection: 'down', + scrollAmount: 1, + } + : { type: actionType, coordinate: { x: 400, y: 200 } }; + const result = await backend.run(action, new AbortController().signal, { + ...DEFAULT_RUN_CONTEXT, + boundAction: boundCoordinateAction(), + }); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'page_target_changed'); + const records = await readRecords(logPath); + assert.equal(toolCalls(records, 'click').length, 0); + assert.equal(toolCalls(records, 'scroll').length, 0); + }); + } + + for (const actionType of ['left_click', 'middle_click', 'scroll'] as const) { + it(`refuses ${actionType} pixel fallback for an unknown process`, async () => { + const { backend, logPath } = makeBackend({ processKind: 'unknown' }); + const action: CuAction = actionType === 'scroll' + ? { + type: 'scroll', + coordinate: { x: 600, y: 400 }, + scrollDirection: 'down', + scrollAmount: 1, + } + : { type: actionType, coordinate: { x: 600, y: 400 } }; + const result = await backend.run(action, new AbortController().signal); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'unsupported_action'); + const records = await readRecords(logPath); + assert.equal(toolCalls(records, 'click').length, 0); + assert.equal(toolCalls(records, 'scroll').length, 0); + }); + } + it('coordinate click stays on fresh same-snapshot pixels over an actionable control', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton' }); const res = await backend.run( @@ -1437,6 +1516,7 @@ describe('cua-driver backend', () => { kind: 'left_click', editable: true, tagName: 'input', + elementToken: 'element-1', focusChanged: true, }, onTrace: (event) => traces.push(event), @@ -1504,6 +1584,38 @@ describe('cua-driver backend', () => { assert.doesNotMatch(JSON.stringify(traces), /Private field label|private value/); }); + it('uses structured set_value verification and updates the fresh observation value', async () => { + const { backend, logPath } = makeBackend({ axRole: 'AXTextField' }); + const signal = new AbortController().signal; + const context = { + sessionId: 'set-value-session', + turnId: 'set-value-turn', + toolCallId: 'set-value', + }; + const observed = await backend.observeApp!({ + app: 'Fixture Window', + includeScreenshot: false, + }, signal, context); + + const result = await backend.runSemantic!({ + type: 'set_value', + observationId: observed.observationId, + elementId: '7', + value: 'updated', + elementIdentity: observed.elements[0]!.identity, + }, signal, { + ...context, + boundAction: boundElementAction(observed, '7'), + }); + + assert.equal(result.outcome.ok, true); + assert.equal(result.observation?.elements[0]?.value, 'updated'); + assert.equal(result.observation?.elements[0]?.identity?.value, 'updated'); + const call = toolCall(await readRecords(logPath), 'set_value'); + assert.equal(call?.element_token, 'snapshot:7'); + assert.equal(call?.element_index, 7); + }); + it('fails closed when the physical-input guard cannot be read', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', @@ -1943,7 +2055,7 @@ describe('cua-driver backend', () => { assert.ok(!methodTrace(await readRecords(logPath)).includes('tools/call:type_text')); }); - it('type after an editable native click uses AXValue and verifies a fresh snapshot', async () => { + it('native type stays unsupported without an actual focused element token', async () => { const { backend, logPath } = makeBackend(); const sig = new AbortController().signal; // Establish the target: click win 77 (device 600,400 → screen 300,200 ∈ win 77). @@ -1951,20 +2063,60 @@ describe('cua-driver backend', () => { assert.equal(click.outcome.ok, true); const typed = await backend.run({ type: 'type', text: 'hello world' } as CuAction, sig); - assert.equal(typed.outcome.ok, true, 'type succeeds once a target is established'); + assert.equal(typed.outcome.ok, false); + if (!typed.outcome.ok) assert.equal(typed.outcome.error, 'unsupported_action'); const records = await readRecords(logPath); - const call = toolCall(records, 'set_value'); - assert.ok(call, 'set_value sent to the agent-clicked native field'); - assert.equal(call!.pid, 4242); - assert.equal(call!.window_id, 77); - assert.equal(call!.element_index, 7); - assert.equal(call!.element_token, 'snapshot:7'); - assert.equal(call!.value, 'hello world'); + assert.equal(toolCalls(records, 'set_value').length, 0); assert.equal(toolCalls(records, 'type_text').length, 0); assert.equal(toolCalls(records, 'press_key').length, 0); - // Red line: the target came from the click, never from a frontmost lookup. - assert.ok(!methodTrace(records).includes('tools/call:list_apps'), 'must never resolve a frontmost pid to type into'); + }); + + it('does not invent a focused token after native layout reflow', async () => { + const { backend, logPath } = makeBackend({ refetchMode: 'moved' }); + const signal = new AbortController().signal; + const click = await backend.run( + { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction, + signal, + ); + assert.equal(click.outcome.ok, true); + + const typed = await backend.run( + { type: 'type', text: 'after reflow' } as CuAction, + signal, + ); + assert.equal(typed.outcome.ok, false); + assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 0); + }); + + it('repeated native type remains unsupported without focused tokens', async () => { + const { backend, logPath } = makeBackend(); + const signal = new AbortController().signal; + await backend.run( + { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction, + signal, + ); + + const first = await backend.run({ type: 'type', text: 'once' } as CuAction, signal); + const retry = await backend.run({ type: 'type', text: 'once' } as CuAction, signal); + + assert.equal(first.outcome.ok, false); + assert.equal(retry.outcome.ok, false); + assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 0); + }); + + it('does not establish native keyboard ownership without a focused element token', async () => { + const { backend, logPath } = makeBackend(); + const signal = new AbortController().signal; + const click = await backend.run( + { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction, + signal, + ); + assert.equal(click.outcome.ok, true); + + const typed = await backend.run({ type: 'type', text: 'blocked' } as CuAction, signal); + assert.equal(typed.outcome.ok, false); + assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 0); }); it('parallel click then type waits for the new click target instead of using the old window', async () => { @@ -1995,11 +2147,10 @@ describe('cua-driver backend', () => { ); const [clicked, typed] = await Promise.all([clickNew, typeNew]); assert.equal(clicked.outcome.ok, true); - assert.equal(typed.outcome.ok, true); + assert.equal(typed.outcome.ok, false); const setCalls = toolCalls(await readRecords(logPath), 'set_value'); - assert.equal(setCalls.length, 1); - assert.equal(setCalls[0]!.window_id, 88); + assert.equal(setCalls.length, 0); }); it('type with no AX-addressable editable field fails before any keyboard dispatch', async () => { @@ -2068,6 +2219,7 @@ describe('cua-driver backend', () => { kind: 'left_click', editable: true, tagName: 'textarea', + elementToken: 'element-1', clickEvents: 1, }, }); @@ -2253,6 +2405,7 @@ describe('cua-driver backend', () => { kind: 'left_click', editable: true, tagName: 'textarea', + elementToken: 'element-1', clickEvents: 1, }, }); @@ -2278,6 +2431,7 @@ describe('cua-driver backend', () => { kind: 'left_click', editable: true, tagName: 'textarea', + elementToken: 'element-1', clickEvents: 1, }, }); @@ -2297,7 +2451,38 @@ describe('cua-driver backend', () => { ]); }); - it('native AX text readback mismatch preserves outcome_unknown', async () => { + it('invalidates an Electron element token immediately after document reload', async () => { + let fingerprintReads = 0; + const { backend, logPath } = makeBackend({ + processKind: 'electron', + pageTarget: testPageTarget(), + emptyAx: true, + semanticPointerResult: { + supported: true, + ok: true, + kind: 'left_click', + editable: true, + tagName: 'textarea', + elementToken: 'element-1', + clickEvents: 1, + }, + resolvePageDocumentFingerprint: async () => + fingerprintReads++ === 0 ? 'document-a' : 'document-b', + }); + const signal = new AbortController().signal; + const click = await backend.run( + { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction, + signal, + ); + assert.equal(click.outcome.ok, true); + + const typed = await backend.run({ type: 'type', text: 'must-not-land' } as CuAction, signal); + assert.equal(typed.outcome.ok, false); + if (!typed.outcome.ok) assert.equal(typed.outcome.error, 'page_target_changed'); + assert.equal(businessPageCalls(await readRecords(logPath)).length, 1); + }); + + it('native type remains unsupported before any AX readback mismatch can occur', async () => { const { backend, logPath } = makeBackend({ nativeReadbackValue: 'wrong', }); @@ -2315,13 +2500,12 @@ describe('cua-driver backend', () => { assert.equal(result.outcome.ok, false); if (!result.outcome.ok) { - assert.equal(result.outcome.error, 'outcome_unknown'); - assert.equal(result.outcome.evidence?.path, 'ax'); + assert.equal(result.outcome.error, 'unsupported_action'); } - assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 1); + assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 0); }); - it('native AX text readback request failure preserves outcome_unknown', async () => { + it('native type remains unsupported before any AX readback request', async () => { const { backend, logPath } = makeBackend({ rpcErrAfterTool: 'get_window_state', rpcErrAfterCount: 3, @@ -2340,10 +2524,9 @@ describe('cua-driver backend', () => { assert.equal(result.outcome.ok, false); if (!result.outcome.ok) { - assert.equal(result.outcome.error, 'outcome_unknown'); - assert.equal(result.outcome.evidence?.path, 'ax'); + assert.equal(result.outcome.error, 'unsupported_action'); } - assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 1); + assert.equal(toolCalls(await readRecords(logPath), 'set_value').length, 0); }); it('CDP text inspection request failure preserves outcome_unknown', async () => { @@ -2357,10 +2540,11 @@ describe('cua-driver backend', () => { kind: 'left_click', editable: true, tagName: 'textarea', + elementToken: 'element-1', clickEvents: 1, }, rpcErrAfterTool: 'page', - rpcErrAfterCount: 5, + rpcErrAfterCount: 6, }); const signal = new AbortController().signal; const click = await backend.run( @@ -2435,7 +2619,7 @@ describe('cua-driver backend', () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', rpcErrAfterTool: 'get_window_state', - rpcErrAfterCount: 3, + rpcErrAfterCount: 2, }); const signal = new AbortController().signal; const context = { diff --git a/packages/computer-use/src/__tests__/cua-driver-page-target.test.ts b/packages/computer-use/src/__tests__/cua-driver-page-target.test.ts index 98b7f5abf8..ef9c2438ad 100644 --- a/packages/computer-use/src/__tests__/cua-driver-page-target.test.ts +++ b/packages/computer-use/src/__tests__/cua-driver-page-target.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { - CUA_INSPECT_PREPARED_ELEMENT_SCRIPT, + buildCuaInspectElementTokenScript, buildCuaPrepareElementAtScreenPointScript, buildCuaSemanticPointerActionScript, parseCuaFocusedPageElement, @@ -12,6 +12,12 @@ import { } from '../cua-driver-page-target.js'; const signal = new AbortController().signal; +const lease = { + sessionId: 'session-1', + sessionGeneration: 2, + documentFingerprint: 'document-a', + navigationGeneration: 3, +}; function target(input: Partial = {}): CuaCdpPageTarget { return { @@ -91,7 +97,7 @@ describe('semantic pointer action script', () => { const click = buildCuaSemanticPointerActionScript({ type: 'left_click', screenPoint: { x: 200, y: 300 }, - }); + }, lease); assert.match(click, /actionType = "left_click"/); assert.match(click, /elementFromPoint/); assert.match(click, /element\.click\(\)/); @@ -102,7 +108,7 @@ describe('semantic pointer action script', () => { type: 'left_click_drag', startScreenPoint: { x: 10, y: 20 }, endScreenPoint: { x: 100, y: 20 }, - }); + }, lease); assert.match(drag, /type \|\| ''\)\.toLowerCase\(\) !== 'range'/); assert.match(drag, /dispatchEvent\(new Event\('input'/); assert.match(drag, /range_value_did_not_persist/); @@ -110,21 +116,29 @@ describe('semantic pointer action script', () => { }); it('builds read-only element scripts and parses their JSON result', () => { - const prepare = buildCuaPrepareElementAtScreenPointScript({ x: 10, y: 20 }); + const prepare = buildCuaPrepareElementAtScreenPointScript({ x: 10, y: 20 }, lease); assert.match(prepare, /elementFromPoint/); - assert.match(prepare, /__makaComputerUseTarget/); + assert.match(prepare, /__makaComputerUseElementState/); assert.doesNotMatch(prepare, /\.focus\s*\(/); - assert.match(CUA_INSPECT_PREPARED_ELEMENT_SCRIPT, /__makaComputerUseReadElement/); + const inspect = buildCuaInspectElementTokenScript({ + ...lease, + elementToken: 'element-1', + }); + assert.match(inspect, /__makaComputerUseReadElement/); + assert.match(inspect, /document\.activeElement !== element/); + assert.match(inspect, /\\"sessionId\\":\\"session-1\\"/); assert.deepEqual( parseCuaFocusedPageElement(JSON.stringify({ editable: true, value: 'ready', tagName: 'textarea', + elementToken: 'element-1', })), { editable: true, value: 'ready', tagName: 'textarea', + elementToken: 'element-1', }, ); }); diff --git a/packages/computer-use/src/__tests__/cua-driver-result.test.ts b/packages/computer-use/src/__tests__/cua-driver-result.test.ts index 4ff009887c..83c1b57e84 100644 --- a/packages/computer-use/src/__tests__/cua-driver-result.test.ts +++ b/packages/computer-use/src/__tests__/cua-driver-result.test.ts @@ -27,6 +27,27 @@ describe('normalizeCuaDriverOutcome', () => { ); }); + it('retains structured same-node set_value verification as internal evidence', () => { + assert.deepEqual( + normalizeCuaDriverOutcome(result({ + path: 'ax', + changed: true, + verified: true, + readback_value: 'model-real-ax', + })), + { + ok: true, + tier: 'ax', + verified: true, + evidence: { + path: 'ax', + changed: true, + readbackValue: 'model-real-ax', + }, + }, + ); + }); + it('maps CGEvent unverifiable evidence to an unverified background success', () => { assert.deepEqual( normalizeCuaDriverOutcome(result({ diff --git a/packages/computer-use/src/cua-driver-backend.ts b/packages/computer-use/src/cua-driver-backend.ts index 4f63176daf..07e4161467 100644 --- a/packages/computer-use/src/cua-driver-backend.ts +++ b/packages/computer-use/src/cua-driver-backend.ts @@ -56,8 +56,7 @@ import { type CuaDriverJsonRpcResponse, } from './cua-driver-service.js'; import { - CUA_INSPECT_PREPARED_ELEMENT_SCRIPT, - buildCuaPrepareElementAtScreenPointScript, + buildCuaInspectElementTokenScript, buildCuaSemanticPointerActionScript, parseCuaFocusedPageElement, parseCuaSemanticPointerResult, @@ -65,6 +64,8 @@ import { type CuaSemanticPointerAction, type CuaSemanticPointerResult, type CuaResolvedPageTextTarget, + type CuaPageElementLeaseContext, + type CuaPageElementTokenLease, } from './cua-driver-page-target.js'; import { editableElementAtScreenPoint, @@ -299,6 +300,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc interface KeyboardTarget { window: CuaResolvedWindow; editable: boolean; + pageElement?: CuaPageElementTokenLease; + pageIdentity?: ComputerUsePageIdentity; pageTarget?: CuaResolvedPageTextTarget; } const targetsBySession = new Map(); @@ -316,6 +319,10 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc } const observations = new Map(); const observationIdsBySession = new Map(); + const pageNavigationGenerations = new Map(); let operationQueue = Promise.resolve(); let disposed = false; @@ -354,8 +361,6 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc const structural = [ ...new Set([...elements].map((element) => JSON.stringify({ role: element.role, - label: element.label, - value: element.value, frame: element.frame, depth: element.depth, }))), @@ -770,6 +775,23 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc && left.documentFingerprint === right.documentFingerprint; } + function pageNavigationGeneration( + window: CuaResolvedWindow, + page: ComputerUsePageIdentity, + ): number { + const key = `${window.pid}:${window.windowId}:${page.cdpPort}:${page.pageTargetId}`; + const documentFingerprint = page.documentFingerprint ?? ''; + const current = pageNavigationGenerations.get(key); + if (!current) { + pageNavigationGenerations.set(key, { documentFingerprint, generation: 1 }); + return 1; + } + if (current.documentFingerprint === documentFingerprint) return current.generation; + const generation = current.generation + 1; + pageNavigationGenerations.set(key, { documentFingerprint, generation }); + return generation; + } + async function resolvePageDocumentFingerprint( window: CuaResolvedWindow, target: CuaResolvedPageTextTarget, @@ -928,8 +950,9 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc const outcome = normalizeCuaDriverOutcome(state); if (!outcome.ok) throw new Error(outcome.message); const structured = state?.structuredContent ?? {}; + const candidates = (structured.elements ?? []) as CuaSnapshotElement[]; const elements = new Map>>(); - for (const candidate of (structured.elements ?? []) as CuaSnapshotElement[]) { + for (const candidate of candidates) { const element = normalizeCuaSnapshotElement(candidate); if (!element) continue; const elementId = String(element.element_index); @@ -999,7 +1022,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc height: element.frame.h, }, identity: { - ...(element.element_token ? { token: element.element_token } : {}), + ...(element.element_token ? { elementToken: element.element_token } : {}), + elementIndex: element.element_index, role: element.role, ...(element.label ? { label: element.label } : {}), ...(element.value !== undefined ? { value: element.value } : {}), @@ -1009,42 +1033,6 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }; } - function elementMatchesIdentity( - element: NonNullable>, - identity: CuObservedElement['identity'] | undefined, - original?: NonNullable>, - ): boolean { - if (!identity) return false; - if (element.role !== identity.role) return false; - const label = identity.label?.trim(); - return (!label || element.label === identity.label) - && !!original - && element.depth === original.depth - && element.frame.x === original.frame.x - && element.frame.y === original.frame.y - && element.frame.w === original.frame.w - && element.frame.h === original.frame.h - && element.value === original.value; - } - - function dedupeSemanticElements( - elements: NonNullable>[], - ): NonNullable>[] { - const seen = new Set(); - return elements.filter((element) => { - const key = JSON.stringify({ - role: element.role, - label: element.label, - value: element.value, - frame: element.frame, - depth: element.depth, - }); - if (seen.has(key)) return false; - seen.add(key); - return true; - }); - } - async function validateSemanticElementVisibility( window: CuaResolvedWindow, element: NonNullable>, @@ -1262,7 +1250,13 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc bound: CuaBoundAction | undefined, signal: AbortSignal, start = false, - ): Promise { + ): Promise< + | (CuaResolvedWindow & { + freshElement?: NonNullable>; + }) + | CuRunResult + | undefined + > { if (!bound?.target) return undefined; if (!bound.target.contentFingerprint) { return { @@ -1301,8 +1295,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc pid: validated.pid, window_id: validated.windowId, include_screenshot: false, - max_elements: 0, - max_depth: 0, + max_elements: 500, + max_depth: 25, }, signal); const currentOutcome = normalizeCuaDriverOutcome(currentState); if (!currentOutcome.ok) return { outcome: currentOutcome }; @@ -1338,6 +1332,56 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }, }; } + const currentElements = (currentStructured.elements ?? []) as CuaSnapshotElement[]; + let freshElement: NonNullable> | undefined; + if (bound.sourceElement) { + const matches = currentElements.flatMap((candidate) => { + const element = normalizeCuaSnapshotElement(candidate); + if ( + !element + || element.role !== bound.sourceElement!.role + || element.label !== bound.sourceElement!.label + || element.value !== bound.sourceElement!.value + || element.frame.x !== bound.sourceElement!.frame.x + || element.frame.y !== bound.sourceElement!.frame.y + || element.frame.w !== bound.sourceElement!.frame.width + || element.frame.h !== bound.sourceElement!.frame.height + ) return []; + return [element]; + }); + if (matches.length !== 1) { + return { + outcome: { + ok: false, + error: 'target_changed', + message: 'source actionable element identity changed after observation', + }, + }; + } + freshElement = matches[0]; + if (!freshElement?.element_token) { + return { + outcome: { + ok: false, + error: 'target_changed', + message: 'fresh actionable element is missing an AX element token', + }, + }; + } + } else { + if ( + editableElementAtScreenPoint(currentElements, point.screenPoint) + || elementAtScreenPoint(currentElements, point.screenPoint) + ) { + return { + outcome: { + ok: false, + error: 'target_changed', + message: 'bound coordinate is missing its source actionable element identity', + }, + }; + } + } const windows = await listWindowRecords(signal); const winner = windows .flatMap((window) => { @@ -1395,6 +1439,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc return { ...validated, screenPoint: point.screenPoint, + ...(freshElement ? { freshElement } : {}), }; } @@ -1403,82 +1448,59 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc fallback: { x: number; y: number }, signal: AbortSignal, start = false, - ): Promise { + ): Promise< + | (CuaResolvedWindow & { + freshElement?: NonNullable>; + }) + | CuRunResult + | undefined + > { if (bound) return validateBoundCoordinate(bound, signal, start); return resolveWindowAt(fallback.x, fallback.y, signal); } - async function refetchSemanticElement( + function storedSemanticElement( observation: StoredObservation, action: Exclude, - signal: AbortSignal, - ): Promise< + ): | NonNullable> - | CuRunResult - > { - const state = await actionClient.callTool('get_window_state', { - pid: observation.window.pid, - window_id: observation.window.windowId, - include_screenshot: false, - max_elements: 500, - max_depth: 25, - }, signal); - const outcome = normalizeCuaDriverOutcome(state); - if (!outcome.ok) return { outcome }; - const fresh = dedupeSemanticElements( - ((state?.structuredContent?.elements ?? []) as CuaSnapshotElement[]) - .flatMap((candidate) => { - const element = normalizeCuaSnapshotElement(candidate); - return element ? [element] : []; - }), - ); + | CuRunResult { const original = observation.elements.get(action.elementId); - const identity = action.elementIdentity ?? ( - original - ? { - ...(original.element_token ? { token: original.element_token } : {}), - role: original.role, - ...(original.label ? { label: original.label } : {}), - ...(original.value !== undefined ? { value: original.value } : {}), - } - : undefined - ); - if (!identity) { + if (!original) { + return { + outcome: { + ok: false, + error: 'stale_frame', + message: 'semantic element is missing from the consumed observation', + }, + }; + } + if (!original.element_token) { return { outcome: { ok: false, error: 'stale_frame', - message: 'semantic element identity is unavailable', + message: 'observed semantic element is missing an AX element token', }, }; } - const identityMatches = fresh.filter((candidate) => - candidate.role === identity.role - && (!identity.label?.trim() || candidate.label === identity.label) + const supplied = action.elementIdentity; + if ( + supplied && ( - identity.value === undefined - || candidate.value === identity.value - )); - if (identityMatches.length > 1) { + supplied.elementToken !== original.element_token + || supplied.elementIndex !== original.element_index + ) + ) { return { outcome: { ok: false, error: 'stale_frame', - message: 'semantic element identity is ambiguous in the fresh observation', + message: 'semantic action identity does not match the consumed observation', }, }; } - const matches = identityMatches.filter( - (candidate) => elementMatchesIdentity(candidate, identity, original), - ); - if (matches.length === 1) return matches[0]!; - return { - outcome: { - ok: false, - error: 'stale_frame', - message: 'semantic element is missing from the fresh observation', - }, - }; + return original; } function targetForContext(context: CuRunContext): KeyboardTarget | undefined { @@ -1521,70 +1543,18 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc message: 'background text input requires an AX-addressable editable field', }; } - const snapshot = await snapshotTarget(target.window, signal); - const element = editableElementAtScreenPoint(snapshot.elements, target.window.screenPoint); - if (!element) { - return { - ok: false, - error: 'unsupported_action', - message: 'editable field was not present in the fresh AX snapshot', - }; - } - if (element.value && element.value !== text) { - return { - ok: false, - error: 'unsupported_action', - message: 'background AX fill refuses to overwrite a non-empty field', - }; - } - if (element.value === text) { - return { - ok: true, - tier: 'ax', - verified: true, - evidence: { path: 'ax', effect: 'confirmed' }, - }; - } - const intervention = await physicalInputFailure(); - if (intervention) return intervention.outcome; - const setResult = await actionClient.callTool( - 'set_value', - { - pid: target.window.pid, - window_id: target.window.windowId, - element_index: element.element_index, - ...(element.element_token ? { element_token: element.element_token } : {}), - value: text, - }, - signal, - ); - if (setResult?.isError) return normalizeCuaDriverOutcome(setResult); - let after: TargetSnapshot; - try { - after = await snapshotTarget(target.window, signal); - } catch { - return deliveredVerificationFailure( - 'AXValue write', - 'ax', - ).outcome; - } - const verified = editableElementAtScreenPoint( - after.elements, - target.window.screenPoint, - )?.value === text; - return verified - ? { - ok: true, - tier: 'ax', - verified: true, - evidence: { path: 'ax', effect: 'confirmed' }, - } - : { - ok: false, - error: 'outcome_unknown', - message: 'AXValue write could not be confirmed by a fresh snapshot', - evidence: { path: 'ax', effect: 'unverifiable' }, - }; + return { + ok: false, + error: 'unsupported_action', + message: 'background native text input requires an actual focused AX element token', + }; + } + + function isEditableRole(role: string): boolean { + return role === 'AXComboBox' + || role === 'AXSearchField' + || role === 'AXTextArea' + || role === 'AXTextField'; } async function fillElectronPageTarget( @@ -1599,6 +1569,15 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc message: 'background Electron text requires a verified text-editable click target', }; } + const pageElement = target.pageElement; + const observedPage = target.pageIdentity; + if (!pageElement || !observedPage) { + return { + ok: false, + error: 'unsupported_action', + message: 'background Electron text requires a document-bound clicked element token', + }; + } const pageTarget = target.pageTarget ?? await ( opts.resolvePageTextTarget ?? ((input) => resolveCuaPageTextTarget(input)) )({ @@ -1613,6 +1592,25 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc message: 'Electron background text requires a unique, already-listening CDP page target', }; } + const currentPage = await pageIdentity(target.window, pageTarget, signal); + if (!currentPage || !samePage(observedPage, currentPage)) { + return { + ok: false, + error: 'page_target_changed', + message: 'Electron document changed after editable ownership was established', + }; + } + const navigationGeneration = pageNavigationGeneration(target.window, currentPage); + if ( + pageElement.documentFingerprint !== currentPage.documentFingerprint + || pageElement.navigationGeneration !== navigationGeneration + ) { + return { + ok: false, + error: 'page_target_changed', + message: 'Electron element token expired after navigation or reload', + }; + } const executePageScript = async (javascript: string) => { const response = await actionClient.callTool( 'page', @@ -1633,7 +1631,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc return { response, element: parseCuaFocusedPageElement(text) }; }; const prepared = await executePageScript( - buildCuaPrepareElementAtScreenPointScript(target.window.screenPoint), + buildCuaInspectElementTokenScript(pageElement), ); if (prepared.response?.isError) return normalizeCuaDriverOutcome(prepared.response); const before = prepared.element; @@ -1676,9 +1674,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc if (result?.isError) return normalizeCuaDriverOutcome(result); let inspected: Awaited>; try { - inspected = await executePageScript( - CUA_INSPECT_PREPARED_ELEMENT_SCRIPT, - ); + inspected = await executePageScript(buildCuaInspectElementTokenScript(pageElement)); } catch { return deliveredVerificationFailure( 'CDP Input.insertText', @@ -1712,15 +1708,38 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc window: CuaResolvedWindow, signal: AbortSignal, toolCallId: string, + context: CuRunContext, boundPage?: ComputerUsePageIdentity, ): Promise<{ handled: boolean; outcome?: CuRunResult['outcome']; result?: CuaSemanticPointerResult; pageTarget?: CuaResolvedPageTextTarget; + pageIdentity?: ComputerUsePageIdentity; + pageElement?: CuaPageElementTokenLease; }> { const processKind = await (opts.classifyProcess ?? classifyMacProcess)(window.pid); + if (processKind === 'unknown') { + return { + handled: true, + outcome: { + ok: false, + error: 'unsupported_action', + message: 'target process type is unknown; pixel fallback is refused', + }, + }; + } if (processKind !== 'electron') return { handled: false }; + if (context.boundAction?.target && !boundPage) { + return { + handled: true, + outcome: { + ok: false, + error: 'page_target_changed', + message: 'bound Electron action is missing observed page identity', + }, + }; + } const resolvePageTextTarget = opts.resolvePageTextTarget ?? ((input) => resolveCuaPageTextTarget(input)); const pageTarget = await resolvePageTextTarget({ @@ -1753,7 +1772,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }; } const currentPage = await pageIdentity(window, pageTarget, signal); - if (boundPage && !samePage(boundPage, currentPage)) { + if (!currentPage || (boundPage && !samePage(boundPage, currentPage))) { return { handled: true, outcome: { @@ -1763,6 +1782,12 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }, }; } + const leaseContext: CuaPageElementLeaseContext = { + sessionId: context.sessionId, + sessionGeneration: sessionGenerations.get(context.sessionId) ?? 0, + documentFingerprint: currentPage.documentFingerprint ?? '', + navigationGeneration: pageNavigationGeneration(window, currentPage), + }; const intervention = await physicalInputFailure(); if (intervention) return { handled: true, outcome: intervention.outcome }; @@ -1781,7 +1806,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc pid: window.pid, window_id: window.windowId, action: 'execute_javascript', - javascript: buildCuaSemanticPointerActionScript(action), + javascript: buildCuaSemanticPointerActionScript(action, leaseContext), cdp_port: pageTarget.port, target_url_contains: pageTarget.targetUrlContains, }, @@ -1863,7 +1888,64 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc tool: 'page', outcome: traceOutcome(outcome), }); - return { handled: true, outcome, result, pageTarget }; + return { + handled: true, + outcome, + result, + pageTarget, + pageIdentity: currentPage, + ...(result.elementToken + ? { pageElement: { ...leaseContext, elementToken: result.elementToken } } + : {}), + }; + } + + async function compatibilityPointerBoundary( + window: CuaResolvedWindow, + boundPage: ComputerUsePageIdentity | undefined, + requireBoundPage: boolean, + signal: AbortSignal, + ): Promise { + const processKind = await (opts.classifyProcess ?? classifyMacProcess)(window.pid); + if (processKind === 'unknown') { + return { + outcome: { + ok: false, + error: 'unsupported_action', + message: 'target process type is unknown; pixel fallback is refused', + }, + }; + } + if (processKind !== 'electron') return undefined; + if (requireBoundPage && !boundPage) { + return { + outcome: { + ok: false, + error: 'page_target_changed', + message: 'bound Electron action is missing observed page identity', + }, + }; + } + const pageTarget = await ( + opts.resolvePageTextTarget ?? ((input) => resolveCuaPageTextTarget(input)) + )({ + pid: window.pid, + ...(window.title ? { windowTitle: window.title } : {}), + signal, + }); + const currentPage = pageTarget + ? await pageIdentity(window, pageTarget, signal) + : undefined; + if (!boundPage || !samePage(boundPage, currentPage)) { + return { + outcome: { + ok: false, + error: 'page_target_changed', + message: 'Electron page identity changed before pixel fallback', + }, + }; + } + return undefined; } return { @@ -1973,19 +2055,19 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }, }; } - const refetched = await refetchSemanticElement(observation, action, signal); - if ('outcome' in refetched) return refetched; + const observedElement = storedSemanticElement(observation, action); + if ('outcome' in observedElement) return observedElement; const visibilityFailure = await validateSemanticElementVisibility( validated, - refetched, + observedElement, signal, ); if (visibilityFailure) return visibilityFailure; const args = { pid: validated.pid, window_id: validated.windowId, - element_index: refetched.element_index, - ...(refetched.element_token ? { element_token: refetched.element_token } : {}), + element_index: observedElement.element_index, + element_token: observedElement.element_token, }; const intervention = await physicalInputFailure(); if (intervention) return intervention; @@ -2008,6 +2090,17 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc } const outcome = normalizeCuaDriverOutcome(result); if (!outcome.ok) return { outcome }; + const structured = result?.structuredContent ?? {}; + if ( + action.type === 'set_value' + && ( + typeof structured.changed !== 'boolean' + || structured.verified !== true + || typeof structured.readback_value !== 'string' + ) + ) { + return deliveredVerificationFailure(action.type, 'ax'); + } let fresh: CuObservation; try { fresh = await observeResolvedWindow( @@ -2126,12 +2219,15 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc win, signal, context.toolCallId, + context, context.boundAction?.target?.page, ); if (semantic.handled && semantic.outcome) { if ( semantic.outcome.ok && action.type === 'left_click' + && semantic.pageElement + && semantic.pageIdentity && (sessionGenerations.get(context.sessionId) ?? 0) === sessionGeneration ) { targetsBySession.set(context.sessionId, { @@ -2139,6 +2235,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc target: { window: win, editable: semantic.result?.editable === true, + ...(semantic.pageElement ? { pageElement: semantic.pageElement } : {}), + ...(semantic.pageIdentity ? { pageIdentity: semantic.pageIdentity } : {}), ...(semantic.pageTarget ? { pageTarget: semantic.pageTarget } : {}), }, }); @@ -2146,6 +2244,56 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc return { outcome: semantic.outcome, resolvedScreenPoint: win.screenPoint }; } } + if ( + action.type === 'left_click' + && context.boundAction?.sourceElement + ) { + const freshElement = win.freshElement; + if (!freshElement?.element_token) { + return { + outcome: { + ok: false, + error: 'target_changed', + message: 'fresh actionable element is missing an AX element token', + }, + }; + } + const intervention = await physicalInputFailure(); + if (intervention) return intervention; + trace({ + type: 'dispatch', + toolCallId: context.toolCallId, + actionType: action.type, + tool: 'click', + pid: win.pid, + windowId: win.windowId, + address: 'ax', + }); + const result = await actionClient.callTool('click', { + pid: win.pid, + window_id: win.windowId, + element_index: freshElement.element_index, + element_token: freshElement.element_token, + }, signal); + const outcome = normalizeCuaDriverOutcome(result); + trace({ + type: 'outcome', + toolCallId: context.toolCallId, + actionType: action.type, + tool: 'click', + outcome: traceOutcome(outcome), + }); + return { outcome, resolvedScreenPoint: win.screenPoint }; + } + if (action.type === 'middle_click' || action.type === 'triple_click') { + const boundaryFailure = await compatibilityPointerBoundary( + win, + context.boundAction?.target?.page, + context.boundAction?.target !== undefined, + signal, + ); + if (boundaryFailure) return boundaryFailure; + } if (opts.allowCompatibilityInputDispatch !== true) { return compatibilityInputBlocked(action.type); } @@ -2240,26 +2388,10 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc tool: toolName, outcome: traceOutcome(outcome), }); - if ( - outcome.ok - && action.type === 'left_click' - && (sessionGenerations.get(context.sessionId) ?? 0) === sessionGeneration - ) { - targetsBySession.set(context.sessionId, { - turnId: context.turnId, - target: { - window: win, - editable: editableElement !== undefined, - }, - }); - } return { outcome, resolvedScreenPoint: win.screenPoint }; } } case 'scroll': { - if (opts.allowCompatibilityInputDispatch !== true) { - return compatibilityInputBlocked(action.type); - } // Scroll REQUIRES a pid and posts via scroll_wheel_at_xy → post_to_pid // (no cursor warp — the warp only exists in the empty-desktop click path). // Resolve the window under the point and scroll it window-locally; fail @@ -2279,6 +2411,16 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }, }; } + const boundaryFailure = await compatibilityPointerBoundary( + win, + context.boundAction?.target?.page, + context.boundAction?.target !== undefined, + signal, + ); + if (boundaryFailure) return boundaryFailure; + if (opts.allowCompatibilityInputDispatch !== true) { + return compatibilityInputBlocked(action.type); + } { let snapshot: TargetSnapshot; try { @@ -2373,6 +2515,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc from, signal, context.toolCallId, + context, context.boundAction?.target?.page, ); if (semantic.handled && semantic.outcome) { @@ -2634,6 +2777,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc observations.clear(); observationIdsBySession.clear(); sessionGenerations.clear(); + pageNavigationGenerations.clear(); const errors: unknown[] = []; for (const client of [actionClient, captureClient]) { try { diff --git a/packages/computer-use/src/cua-driver-page-target.ts b/packages/computer-use/src/cua-driver-page-target.ts index 016b43694d..bcff402e43 100644 --- a/packages/computer-use/src/cua-driver-page-target.ts +++ b/packages/computer-use/src/cua-driver-page-target.ts @@ -13,6 +13,18 @@ export interface CuaFocusedPageElement { value: string; tagName: string; inputType?: string; + elementToken?: string; +} + +export interface CuaPageElementLeaseContext { + sessionId: string; + sessionGeneration: number; + documentFingerprint: string; + navigationGeneration: number; +} + +export interface CuaPageElementTokenLease extends CuaPageElementLeaseContext { + elementToken: string; } export interface CuaResolvedPageTextTarget { @@ -41,6 +53,7 @@ export interface CuaSemanticPointerResult { editable?: boolean; tagName?: string; inputType?: string; + elementToken?: string; clickEvents?: number; doubleClickEvents?: number; contextMenuEvents?: number; @@ -58,12 +71,6 @@ export interface CuaPageTargetResolverDeps { fetchTargets?: (port: number, signal: AbortSignal) => Promise; } -export const CUA_INSPECT_PREPARED_ELEMENT_SCRIPT = `(() => { - const element = globalThis.__makaComputerUseTarget; - if (!element) return JSON.stringify({ editable: false, value: '', tagName: '' }); - return JSON.stringify(globalThis.__makaComputerUseReadElement(element)); -})()`; - const TEXT_INPUT_TYPES = [ 'email', 'number', @@ -73,6 +80,71 @@ const TEXT_INPUT_TYPES = [ 'url', ] as const; +function pageElementHelperBootstrap(context: CuaPageElementLeaseContext): string { + const leaseContext: CuaPageElementLeaseContext = { + sessionId: context.sessionId, + sessionGeneration: context.sessionGeneration, + documentFingerprint: context.documentFingerprint, + navigationGeneration: context.navigationGeneration, + }; + return ` + const textInputTypes = new Set(${JSON.stringify(TEXT_INPUT_TYPES)}); + const leaseKey = ${JSON.stringify(JSON.stringify(leaseContext))}; + if ( + !globalThis.__makaComputerUseElementState + || globalThis.__makaComputerUseElementState.leaseKey !== leaseKey + ) { + globalThis.__makaComputerUseElementState = { + leaseKey, + sequence: 0, + elements: new Map() + }; + } + globalThis.__makaComputerUseReadElement = (element) => { + const tagName = String(element?.tagName || '').toLowerCase(); + const inputType = tagName === 'input' ? String(element.type || 'text').toLowerCase() : ''; + const editable = !element?.disabled + && !element?.readOnly + && element?.getAttribute?.('aria-disabled') !== 'true' + && ( + tagName === 'textarea' + || (tagName === 'input' && textInputTypes.has(inputType)) + || element?.isContentEditable === true + ); + const value = tagName === 'input' || tagName === 'textarea' + ? String(element.value || '') + : element?.isContentEditable === true + ? String(element.textContent || '') + : ''; + return { editable, value, tagName, inputType }; + }; + const rememberElement = (element) => { + const state = globalThis.__makaComputerUseElementState; + state.sequence += 1; + const elementToken = leaseKey + ':' + String(state.sequence); + state.elements.set(elementToken, element); + return elementToken; + }; + `; +} + +export function buildCuaInspectElementTokenScript( + lease: CuaPageElementTokenLease, +): string { + return `(() => { + ${pageElementHelperBootstrap(lease)} + const elementToken = ${JSON.stringify(lease.elementToken)}; + const element = globalThis.__makaComputerUseElementState.elements.get(elementToken); + if (!element || !element.isConnected || document.activeElement !== element) { + return JSON.stringify({ editable: false, value: '', tagName: '', elementToken }); + } + return JSON.stringify({ + ...globalThis.__makaComputerUseReadElement(element), + elementToken + }); + })()`; +} + export async function resolveCuaPageTextTarget( input: { pid: number; @@ -145,27 +217,12 @@ function uniqueUrlHint( return target.url; } -export function buildCuaPrepareElementAtScreenPointScript(screenPoint: CuPoint): string { +export function buildCuaPrepareElementAtScreenPointScript( + screenPoint: CuPoint, + lease: CuaPageElementLeaseContext, +): string { return `(() => { - const textInputTypes = new Set(${JSON.stringify(TEXT_INPUT_TYPES)}); - globalThis.__makaComputerUseReadElement = (element) => { - const tagName = String(element?.tagName || '').toLowerCase(); - const inputType = tagName === 'input' ? String(element.type || 'text').toLowerCase() : ''; - const editable = !element?.disabled - && !element?.readOnly - && element?.getAttribute?.('aria-disabled') !== 'true' - && ( - tagName === 'textarea' - || (tagName === 'input' && textInputTypes.has(inputType)) - || element?.isContentEditable === true - ); - const value = tagName === 'input' || tagName === 'textarea' - ? String(element.value || '') - : element?.isContentEditable === true - ? String(element.textContent || '') - : ''; - return { editable, value, tagName, inputType }; - }; + ${pageElementHelperBootstrap(lease)} const chromeLeft = Math.max(0, (window.outerWidth - window.innerWidth) / 2); const chromeTop = Math.max(0, window.outerHeight - window.innerHeight - chromeLeft); const viewportX = ${JSON.stringify(screenPoint.x)} - window.screenX - chromeLeft; @@ -175,16 +232,19 @@ export function buildCuaPrepareElementAtScreenPointScript(screenPoint: CuPoint): element = element.parentElement; } if (!element) { - globalThis.__makaComputerUseTarget = undefined; return JSON.stringify({ editable: false, value: '', tagName: '' }); } - globalThis.__makaComputerUseTarget = element; - return JSON.stringify(globalThis.__makaComputerUseReadElement(element)); + const elementToken = rememberElement(element); + return JSON.stringify({ + ...globalThis.__makaComputerUseReadElement(element), + elementToken + }); })()`; } export function buildCuaSemanticPointerActionScript( action: CuaSemanticPointerAction, + lease: CuaPageElementLeaseContext, ): string { const start = action.type === 'left_click_drag' ? action.startScreenPoint @@ -194,7 +254,7 @@ export function buildCuaSemanticPointerActionScript( : action.screenPoint; return `(async () => { const actionType = ${JSON.stringify(action.type)}; - const textInputTypes = new Set(${JSON.stringify(TEXT_INPUT_TYPES)}); + ${pageElementHelperBootstrap(lease)} const chromeLeft = Math.max(0, (window.outerWidth - window.innerWidth) / 2); const chromeTop = Math.max(0, window.outerHeight - window.innerHeight - chromeLeft); const viewportPoint = (screenX, screenY) => ({ @@ -243,6 +303,7 @@ export function buildCuaSemanticPointerActionScript( const checkedChanged = beforeChecked !== undefined && element.checked !== beforeChecked; const valueChanged = beforeValue !== undefined && String(element.value ?? '') !== beforeValue; const focusedEditable = editable && document.activeElement === element; + const elementToken = focusedEditable ? rememberElement(element) : undefined; const ok = focusedEditable || checkedChanged || valueChanged || mutations > 0; return JSON.stringify({ supported: true, @@ -260,6 +321,7 @@ export function buildCuaSemanticPointerActionScript( editable, tagName, inputType, + ...(elementToken ? { elementToken } : {}), clickEvents, mutations, ...(typeof element.checked === 'boolean' ? { checked: element.checked } : {}), @@ -431,6 +493,7 @@ export function parseCuaSemanticPointerResult( ...(typeof result.editable === 'boolean' ? { editable: result.editable } : {}), ...(typeof result.tagName === 'string' ? { tagName: result.tagName } : {}), ...(typeof result.inputType === 'string' ? { inputType: result.inputType } : {}), + ...(typeof result.elementToken === 'string' ? { elementToken: result.elementToken } : {}), ...(typeof result.clickEvents === 'number' ? { clickEvents: result.clickEvents } : {}), ...(typeof result.doubleClickEvents === 'number' ? { doubleClickEvents: result.doubleClickEvents } : {}), ...(typeof result.contextMenuEvents === 'number' ? { contextMenuEvents: result.contextMenuEvents } : {}), @@ -531,5 +594,6 @@ function focusedPageElement(value: unknown): CuaFocusedPageElement { value: typeof result.value === 'string' ? result.value : '', tagName: typeof result.tagName === 'string' ? result.tagName : '', ...(typeof result.inputType === 'string' ? { inputType: result.inputType } : {}), + ...(typeof result.elementToken === 'string' ? { elementToken: result.elementToken } : {}), }; } diff --git a/packages/computer-use/src/cua-driver-result.ts b/packages/computer-use/src/cua-driver-result.ts index d9a9046a48..f51a07a1e6 100644 --- a/packages/computer-use/src/cua-driver-result.ts +++ b/packages/computer-use/src/cua-driver-result.ts @@ -36,12 +36,24 @@ function dispatchEvidence( const reason = typeof structuredContent.reason === 'string' ? structuredContent.reason : undefined; - return path === undefined && effect === undefined && reason === undefined + const changed = typeof structuredContent.changed === 'boolean' + ? structuredContent.changed + : undefined; + const readbackValue = typeof structuredContent.readback_value === 'string' + ? structuredContent.readback_value + : undefined; + return path === undefined + && effect === undefined + && reason === undefined + && changed === undefined + && readbackValue === undefined ? undefined : { ...(path === undefined ? {} : { path }), ...(effect === undefined ? {} : { effect }), ...(reason === undefined ? {} : { reason }), + ...(changed === undefined ? {} : { changed }), + ...(readbackValue === undefined ? {} : { readbackValue }), }; } diff --git a/packages/computer-use/src/index.ts b/packages/computer-use/src/index.ts index 8c3ac94124..eb2c8ce71a 100644 --- a/packages/computer-use/src/index.ts +++ b/packages/computer-use/src/index.ts @@ -2,7 +2,10 @@ export { selectComputerUseBackend } from './select-backend.js'; export type { CuBackendId, SelectedComputerUseBackend } from './select-backend.js'; export { createCuaDriverBackend } from './cua-driver-backend.js'; -export type { CuaDriverBackendOptions, CuaDriverTraceEvent } from './cua-driver-backend.js'; +export type { + CuaDriverBackendOptions, + CuaDriverTraceEvent, +} from './cua-driver-backend.js'; export { normalizeCuaDriverOutcome } from './cua-driver-result.js'; export type { JsonRpcToolResult } from './cua-driver-result.js'; export { @@ -27,13 +30,15 @@ export { resolveCuaPageTextTarget } from './cua-driver-page-target.js'; export type { CuaCdpPageTarget, CuaFocusedPageElement, + CuaPageElementLeaseContext, + CuaPageElementTokenLease, CuaPageTargetResolverDeps, CuaResolvedPageTextTarget, CuaSemanticPointerAction, CuaSemanticPointerResult, } from './cua-driver-page-target.js'; export { - CUA_INSPECT_PREPARED_ELEMENT_SCRIPT, + buildCuaInspectElementTokenScript, buildCuaPrepareElementAtScreenPointScript, buildCuaSemanticPointerActionScript, parseCuaFocusedPageElement, diff --git a/packages/core/src/computer-use.ts b/packages/core/src/computer-use.ts index 4e419eb318..6a29cc9167 100644 --- a/packages/core/src/computer-use.ts +++ b/packages/core/src/computer-use.ts @@ -98,12 +98,22 @@ export interface ComputerUseWindowIdentity { page?: ComputerUsePageIdentity; } +export interface ComputerUseElementTargetIdentity { + elementToken?: string; + elementIndex: number; + role: string; + label?: string; + value?: string; + frame: ComputerUseRect; +} + export interface ComputerUseObservationIdentity extends ComputerUseFrameIdentity { capturedAt: number; screenshotWidthPx?: number; screenshotHeightPx?: number; displays: ComputerUseDisplayIdentity[]; target: ComputerUseWindowIdentity; + elements?: ComputerUseElementTargetIdentity[]; } export interface ComputerUseBoundAction extends ComputerUseFrameIdentity { @@ -116,6 +126,7 @@ export interface ComputerUseBoundAction extends ComputerUseFrameIdentity { windowCoordinate?: CuPoint; windowStartCoordinate?: CuPoint; coordinateSpace?: 'window-screenshot-local'; + sourceElement?: ComputerUseElementTargetIdentity; } export const CU_SCROLL_DIRECTIONS = ['up', 'down', 'left', 'right'] as const; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 908c00df3a..bae5860071 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -326,6 +326,7 @@ export type { ComputerUseDispatchEvidence, ComputerUseDispatchTier, ComputerUseDisplayIdentity, + ComputerUseElementTargetIdentity, ComputerUseEffect, ComputerUseErrorCode, ComputerUseBoundAction, diff --git a/packages/runtime/src/__tests__/computer-use-model-loop.test.ts b/packages/runtime/src/__tests__/computer-use-model-loop.test.ts index 82bab148cf..bde4168f47 100644 --- a/packages/runtime/src/__tests__/computer-use-model-loop.test.ts +++ b/packages/runtime/src/__tests__/computer-use-model-loop.test.ts @@ -276,6 +276,7 @@ function fakeComputerBackend( label: 'CUA Lab Set Value Field', value: value.current, identity: { + elementIndex: 7, role: 'AXTextField', label: 'CUA Lab Set Value Field', value: value.current, diff --git a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts index 4a9e6b1da4..b1a9b12f48 100644 --- a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts +++ b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts @@ -138,6 +138,7 @@ function fakeSemanticBackend(value: { current: string }): CuDispatchBackend { label: 'CUA Lab Set Value Field', value: value.current, identity: { + elementIndex: 7, role: 'AXTextField', label: 'CUA Lab Set Value Field', value: value.current, diff --git a/packages/runtime/src/__tests__/computer-use-tools.test.ts b/packages/runtime/src/__tests__/computer-use-tools.test.ts index b80c9d6d42..54d455bc3b 100644 --- a/packages/runtime/src/__tests__/computer-use-tools.test.ts +++ b/packages/runtime/src/__tests__/computer-use-tools.test.ts @@ -71,7 +71,12 @@ function observation(over: Partial = {}): CuObservation { elementId: '5', role: 'AXButton', label: 'Continue', - identity: { token: 'button-token', role: 'AXButton', label: 'Continue' }, + identity: { + elementToken: 'button-token', + elementIndex: 5, + role: 'AXButton', + label: 'Continue', + }, }], screenshot: { base64: 'AA==', @@ -736,7 +741,8 @@ describe('buildComputerUseTools — the `maka_computer` MakaTool', () => { assert.equal((seen[0].action as { observationId: string }).observationId, 'backend-obs-1'); assert.deepEqual((seen[0].action as { elementIdentity?: unknown }).elementIdentity, { - token: 'button-token', + elementToken: 'button-token', + elementIndex: 5, role: 'AXButton', label: 'Continue', }); @@ -981,7 +987,8 @@ describe('buildComputerUseTools — the `maka_computer` MakaTool', () => { } as never, ctx()); assert.deepEqual((seen[0] as { elementIdentity?: unknown }).elementIdentity, { - token: 'button-token', + elementToken: 'button-token', + elementIndex: 5, role: 'AXButton', label: 'Continue', }); diff --git a/packages/runtime/src/__tests__/cua-frame-state.test.ts b/packages/runtime/src/__tests__/cua-frame-state.test.ts index 402d3b9985..a87dedafdb 100644 --- a/packages/runtime/src/__tests__/cua-frame-state.test.ts +++ b/packages/runtime/src/__tests__/cua-frame-state.test.ts @@ -121,6 +121,14 @@ describe('CuaFrameState', () => { bounds: { x: 100, y: 200, width: 800, height: 600 }, sourceBoundsPx: { x: 0, y: 0, width: 800, height: 600 }, }, + elements: [{ + elementToken: 'snapshot:7', + elementIndex: 7, + role: 'AXTextField', + label: 'Name', + value: '', + frame: { x: 120, y: 220, width: 100, height: 40 }, + }], }); const action: CuAction = { type: 'left_click', @@ -132,6 +140,53 @@ describe('CuaFrameState', () => { assert.equal(bound?.target?.windowId, 7); assert.deepEqual(bound?.windowCoordinate, { x: 25, y: 30 }); assert.equal(bound?.coordinateSpace, 'window-screenshot-local'); + assert.deepEqual(bound?.sourceElement, { + elementToken: 'snapshot:7', + elementIndex: 7, + role: 'AXTextField', + label: 'Name', + value: '', + frame: { x: 120, y: 220, width: 100, height: 40 }, + }); + }); + + test('rejects a bound coordinate whose source element identity is tampered', () => { + const state = createState(); + const observation = state.observe({ + capturedAt: 1, + screenshotWidthPx: 800, + screenshotHeightPx: 600, + displays: [], + target: { + pid: 42, + windowId: 7, + bounds: { x: 100, y: 200, width: 800, height: 600 }, + sourceBoundsPx: { x: 0, y: 0, width: 800, height: 600 }, + }, + elements: [{ + elementToken: 'snapshot:7', + elementIndex: 7, + role: 'AXButton', + label: 'Delete', + frame: { x: 120, y: 220, width: 100, height: 40 }, + }], + }); + const bound = bindCuaActionToObservation(observation, { + type: 'left_click', + coordinate: { x: 25, y: 30 }, + }); + assert.ok(bound); + + assert.deepEqual(state.claimAction({ + ...bound, + sourceElement: { + ...bound.sourceElement!, + label: 'Confirm purchase', + }, + }), { + ok: false, + reason: 'invalid_binding', + }); }); test('rejects a coordinate outside the bound window screenshot', () => { diff --git a/packages/runtime/src/computer-use-tools.ts b/packages/runtime/src/computer-use-tools.ts index 1c3f21c6cc..5071a98294 100644 --- a/packages/runtime/src/computer-use-tools.ts +++ b/packages/runtime/src/computer-use-tools.ts @@ -52,6 +52,8 @@ export interface CuDispatchEvidence { path?: string; effect?: ComputerUseEffect; reason?: string; + changed?: boolean; + readbackValue?: string; } export type CuDispatchOutcome = @@ -95,7 +97,8 @@ export interface CuObservedElement { value?: string; frame?: { x: number; y: number; width: number; height: number }; identity?: { - token?: string; + elementToken?: string; + elementIndex: number; role: string; label?: string; value?: string; @@ -717,6 +720,19 @@ export function buildComputerUseTools(deps: { ...(height !== undefined ? { screenshotHeightPx: height } : {}), displays, target, + elements: observation.elements.flatMap((element) => + element.frame + ? [{ + ...(element.identity?.elementToken + ? { elementToken: element.identity.elementToken } + : {}), + elementIndex: element.identity?.elementIndex ?? Number(element.elementId), + role: element.role, + ...(element.label !== undefined ? { label: element.label } : {}), + ...(element.value !== undefined ? { value: element.value } : {}), + frame: element.frame, + }] + : []), }; } @@ -729,6 +745,7 @@ export function buildComputerUseTools(deps: { elements: observation.elements.map((element) => ({ ...element, identity: element.identity ?? { + elementIndex: Number(element.elementId), role: element.role, ...(element.label ? { label: element.label } : {}), ...(element.value !== undefined ? { value: element.value } : {}), diff --git a/packages/runtime/src/cua-frame-state.ts b/packages/runtime/src/cua-frame-state.ts index 99ac2ccef7..86457f7e1d 100644 --- a/packages/runtime/src/cua-frame-state.ts +++ b/packages/runtime/src/cua-frame-state.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import type { ComputerUseBoundAction, + ComputerUseElementTargetIdentity, ComputerUseFrameIdentity, ComputerUseObservationIdentity, ComputerUseWindowIdentity, @@ -20,6 +21,7 @@ export interface CuaObservationSnapshot { screenshotHeightPx?: number; displays: ComputerUseObservationIdentity['displays']; target: ComputerUseWindowIdentity; + elements?: ComputerUseElementTargetIdentity[]; } export type CuaActionRejectionReason = @@ -197,6 +199,7 @@ export function bindCuaActionToObservation( if ('coordinate' in action) { const end = bindWindowPoint(observation, action.coordinate); if (!end) return undefined; + const sourceElement = elementAtSourcePoint(observation, end); if (action.type === 'left_click_drag') { const start = bindWindowPoint(observation, action.startCoordinate); if (!start) return undefined; @@ -207,6 +210,7 @@ export function bindCuaActionToObservation( windowStartCoordinate: start, windowCoordinate: end, coordinateSpace: 'window-screenshot-local', + ...(sourceElement ? { sourceElement } : {}), }); } return finalizeBoundAction({ @@ -214,11 +218,54 @@ export function bindCuaActionToObservation( sourceCoordinate: end, windowCoordinate: end, coordinateSpace: 'window-screenshot-local', + ...(sourceElement ? { sourceElement } : {}), }); } return base; } +const ACTIONABLE_ROLES = new Set([ + 'AXButton', + 'AXCheckBox', + 'AXComboBox', + 'AXDisclosureTriangle', + 'AXLink', + 'AXMenuBarItem', + 'AXMenuButton', + 'AXMenuItem', + 'AXPopUpButton', + 'AXRadioButton', + 'AXScrollArea', + 'AXSearchField', + 'AXTab', + 'AXTextArea', + 'AXTextField', +]); + +function elementAtSourcePoint( + observation: CuaObservation, + point: CuPoint, +): ComputerUseElementTargetIdentity | undefined { + const bounds = observation.target.bounds; + const sourceBounds = observation.target.sourceBoundsPx; + if (!bounds || !sourceBounds || sourceBounds.width <= 0 || sourceBounds.height <= 0) { + return undefined; + } + const screenPoint = { + x: bounds.x + point.x / sourceBounds.width * bounds.width, + y: bounds.y + point.y / sourceBounds.height * bounds.height, + }; + return observation.elements + ?.filter((element) => + ACTIONABLE_ROLES.has(element.role) + && screenPoint.x >= element.frame.x + && screenPoint.x < element.frame.x + element.frame.width + && screenPoint.y >= element.frame.y + && screenPoint.y < element.frame.y + element.frame.height) + .sort((left, right) => + left.frame.width * left.frame.height - right.frame.width * right.frame.height)[0]; +} + function bindWindowPoint( observation: CuaObservation, point: CuPoint, @@ -259,6 +306,7 @@ function fingerprintBoundAction( action.target.pid, action.target.windowId, action.elementId ?? null, + action.sourceElement ?? null, action.sourceStartCoordinate ?? null, action.sourceCoordinate ?? null, ]);