From 07eb3001b5a3d31671014ee0a25939d2ec1a369a Mon Sep 17 00:00:00 2001 From: Gu Date: Sun, 7 Jun 2026 10:44:28 +0800 Subject: [PATCH] feat(desktop): add guarded browser click execution (Phase 2F-A2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move click from AWAITING_SAFETY_ACTIONS to EXECUTABLE_ACTIONS. - Add getInteractiveEnumerationScript(): deterministic @eN candidate enumeration shared by verify-action-target and execute-click. - Add executeClick IPC handler with semantic safety guards: - Rejects submit_button, reset_button, file_input, password_input, external_link by semanticRole. - Rejects destructive labels (delete/remove/destroy/discard/clear all). - Re-verifies: targetRef, URL, fingerprint, visible, not disabled. - Executes via Electron wc.sendInputEvent (native mouseDown/mouseUp). - Add getInteractiveSnapshot IPC (read-only element enumeration). - Add buildSafetyContextFromElement() helper in desktop-visible-provider. - Add executeDesktopClick() function with pre-execution bridge checks. - Keep type non-executable (still AWAITING_SAFETY, returns failed). - Keep eval/press_key/scroll permanently denied. - No agent-provided JS execution — all scripts are fixed strings. - sendInputEvent is the only mutation path. 174 browser-runtime tests pass. 79 platform tests pass. Co-Authored-By: Claude Opus 4.8 --- apps/desktop/electron/main.cjs | 533 ++++++++++++++++-- apps/desktop/electron/preload.cjs | 2 + .../app/browser-runtime/action-gateway-ui.tsx | 62 +- .../browser-runtime/action-gateway.test.ts | 77 ++- .../src/app/browser-runtime/action-gateway.ts | 2 +- .../desktop-visible-provider.test.ts | 402 ++++++++++++- .../desktop-visible-provider.ts | 300 +++++++++- apps/desktop/src/app/browser-workspace.tsx | 80 ++- apps/desktop/src/global.d.ts | 81 +++ 9 files changed, 1431 insertions(+), 108 deletions(-) diff --git a/apps/desktop/electron/main.cjs b/apps/desktop/electron/main.cjs index 2930a98cd7de..e84f1d8bf966 100644 --- a/apps/desktop/electron/main.cjs +++ b/apps/desktop/electron/main.cjs @@ -5672,22 +5672,224 @@ ipcMain.handle('hermes:browser:is-available', async () => { } }) -// ── Phase 2F-A: Read-only target resolution ──────────────────────────── +// ── Phase 2F-A2: Shared interactive element enumeration ──────────────── // -// This handler performs a PURELY READ-ONLY DOM query to verify that a -// target element matching the agent's safetyContext still exists on the -// current page. It uses a fixed internal script — no agent-supplied JS -// is ever evaluated. It does NOT click, focus, dispatchEvent, or set -// any value. +// Both `verify-action-target` and `get-interactive-snapshot` use the +// SAME deterministic candidate enumeration logic. `@e1` maps to +// `candidates[0]`, `@e2` maps to `candidates[1]`, etc. // -// Security: -// - Fixed script string (not user/agent-provided). -// - The only dynamic input is `targetRef`, validated against /^@e\d+$/. -// - Returns element metadata only — no page mutation. -// - No eval of arbitrary expressions. +// Candidates are interactive elements in DOM order: +// button, a[href], input:not([type="hidden"]), textarea, select, +// [role="button"], [role="link"], [contenteditable="true"], +// [tabindex]:not([tabindex="-1"]) +// +// This is a PURELY READ-ONLY query. No mutation, no event dispatch. +// The script is a fixed string — no user/agent input reaches it. // // @see docs/architecture/desktop-browser-agent-action-safety.md §5.1 +/** @returns {string} Fixed script that enumerates interactive candidates as JSON. */ +function getInteractiveEnumerationScript() { + // Inline so the script string is self-contained (no closure captures). + // The script is treated as trusted content: it was reviewed for safety. + // It does NOT mutate the DOM, dispatch events, or access storage. + + return `(() => { + const SELECTORS = [ + 'button', + 'a[href]', + 'input:not([type="hidden"])', + 'textarea', + 'select', + '[role="button"]', + '[role="link"]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])', + ] + + const seen = new Set() + const candidates = [] + + for (const sel of SELECTORS) { + try { + const nodes = document.querySelectorAll(sel) + for (let i = 0; i < nodes.length; i++) { + const el = nodes[i] + if (seen.has(el)) continue + seen.add(el) + candidates.push(el) + } + } catch (_) { + // Invalid selector (should never happen with fixed list) — skip + } + } + + const currentUrl = window.location.href + const elements = [] + + for (let i = 0; i < candidates.length; i++) { + const el = candidates[i] + const tagName = el.tagName + const rect = el.getBoundingClientRect() + const style = window.getComputedStyle(el) + + const visible = ( + rect.width > 0 && + rect.height > 0 && + style.visibility !== 'hidden' && + style.display !== 'none' && + parseFloat(style.opacity) > 0 + ) + + const disabled = ( + el.disabled === true || + el.getAttribute('aria-disabled') === 'true' || + el.getAttribute('disabled') !== null + ) + + const readOnly = ( + el.readOnly === true || + el.getAttribute('aria-readonly') === 'true' || + el.getAttribute('readonly') !== null + ) + + const inputType = (tagName === 'INPUT' || tagName === 'TEXTAREA' || tagName === 'SELECT') + ? (el.getAttribute('type') || 'text') : null + + const textContent = (el.textContent || '').trim().slice(0, 200) + const ariaLabel = el.getAttribute('aria-label') || null + const id = el.id || null + const name = el.getAttribute('name') || null + const placeholder = el.getAttribute('placeholder') || null + const role = el.getAttribute('role') || null + + // value preview for input-like elements (truncated) + const valuePreview = (el.value !== undefined && el.value !== null) + ? String(el.value).slice(0, 200) : null + + // ── Semantic role classification ───────────────────────────────── + // Used by execute-click to reject high-risk targets. + const tag = tagName.toLowerCase() + const typeAttr = (el.getAttribute('type') || '').toLowerCase() + const text = (el.textContent || '').toLowerCase().trim() + const href = (el.getAttribute('href') || '').trim() + let semanticRole = 'generic' + + if (tag === 'a' && href) { + const isInternal = href.startsWith('#') || href.startsWith('/') || href.startsWith(window.location.origin) + semanticRole = isInternal ? 'internal_link' : 'external_link' + } else if (tag === 'button') { + if (typeAttr === 'submit') semanticRole = 'submit_button' + else if (typeAttr === 'reset') semanticRole = 'reset_button' + else semanticRole = 'button' + } else if (tag === 'input') { + if (typeAttr === 'submit') semanticRole = 'submit_button' + else if (typeAttr === 'file') semanticRole = 'file_input' + else if (typeAttr === 'password') semanticRole = 'password_input' + else if (typeAttr === 'checkbox' || typeAttr === 'radio') semanticRole = 'toggle_input' + else semanticRole = 'text_input' + } else if (tag === 'textarea') { + semanticRole = 'text_input' + } else if (tag === 'select') { + semanticRole = 'select_input' + } else if (role === 'button' || role === 'link') { + semanticRole = role + } + + // ── Destructive action detection ───────────────────────────────── + const DESTRUCTIVE_KEYWORDS = ['delete', 'remove', 'destroy', 'discard', 'clear all', 'reset all'] + const isDestructive = DESTRUCTIVE_KEYWORDS.some(kw => text.includes(kw)) + // ── Medium-risk: settings, preferences, admin, manage ──────────── + const MEDIUM_RISK_KEYWORDS = ['settings', 'preferences', 'admin', 'manage', 'config', 'payment', 'billing'] + const isMediumRisk = MEDIUM_RISK_KEYWORDS.some(kw => text.includes(kw)) + + // High-risk: submit, reset, file, password, destructive, external links + const highRisk = ( + semanticRole === 'submit_button' + || semanticRole === 'reset_button' + || semanticRole === 'file_input' + || semanticRole === 'password_input' + || semanticRole === 'external_link' + || isDestructive + ) + + elements.push({ + ref: '@e' + (i + 1), + tagName: tagName.toLowerCase(), + role: role, + semanticRole: semanticRole, + highRisk: highRisk, + isDestructive: isDestructive, + isMediumRisk: isMediumRisk, + textContent: textContent, + ariaLabel: ariaLabel, + id: id, + name: name, + inputType: inputType, + placeholder: placeholder, + valuePreview: valuePreview, + href: href || null, + boundingBox: { + x: Math.round(rect.x), + y: Math.round(rect.y), + w: Math.round(rect.width), + h: Math.round(rect.height), + }, + visible: visible, + disabled: disabled, + readOnly: readOnly, + fingerprint: { + tagName: tagName.toLowerCase(), + textContent: textContent, + id: id, + name: name, + inputType: inputType, + ariaLabel: ariaLabel, + rect: { + x: Math.round(rect.x), + y: Math.round(rect.y), + w: Math.round(rect.width), + h: Math.round(rect.height), + }, + }, + }) + } + + return { currentUrl: currentUrl, elements: elements } + })()` +} + +// ── Read-only: Get interactive snapshot ──────────────────────────────── + +ipcMain.handle('hermes:browser:get-interactive-snapshot', async () => { + try { + const view = getBrowserView() + const wc = view.webContents + const result = await wc.executeJavaScript(getInteractiveEnumerationScript()) + return { + ok: true, + capturedAt: new Date().toISOString(), + ...result, + } + } catch (error) { + return { + ok: false, + capturedAt: new Date().toISOString(), + currentUrl: '', + elements: [], + error: error?.message || String(error), + } + } +}) + +// ── Read-only: Verify action target ───────────────────────────────────── +// +// Resolves @eN by re-running the SAME interactive candidate enumeration +// and picking candidates[N-1]. Falls back to data-agent-ref / aria- +// describedby for pages that have their own snapshot-ref attributes (e.g. +// agent-browser-driven CDP sessions), but the PRIMARY path is the +// deterministic enumeration shared with get-interactive-snapshot. + ipcMain.handle('hermes:browser:verify-action-target', async (_event, payload) => { try { const targetRef = String(payload?.targetRef || '').trim() @@ -5703,29 +5905,53 @@ ipcMain.handle('hermes:browser:verify-action-target', async (_event, payload) => } } + const targetIndex = Number.parseInt(targetRef.slice(2), 10) - 1 + if (targetIndex < 0 || !Number.isFinite(targetIndex)) { + return { + found: false, + reason: 'invalid_target_ref', + currentUrl: '', + urlMatchesOrigin: false, + } + } + const view = getBrowserView() const wc = view.webContents - // ── Resolve element by ref ────────────────────────────────────────── - // The ref @eN maps to an element with a data-agent-ref attribute or - // an aria attribute set by the accessibility snapshot system. - // We use a fixed script that queries the DOM without mutation. - const result = await wc.executeJavaScript(` + // ── Primary path: deterministic candidate enumeration ────────────── + const snapshot = await wc.executeJavaScript(getInteractiveEnumerationScript()) + const candidates = snapshot?.elements || [] + const currentUrl = snapshot?.currentUrl || wc.getURL() + + if (targetIndex < candidates.length) { + const el = candidates[targetIndex] + + return { + found: true, + reason: null, + currentUrl: currentUrl, + urlMatchesOrigin: currentUrl === originUrl, + elementFingerprint: el.fingerprint, + boundingBox: el.boundingBox, + visible: el.visible, + disabled: el.disabled, + readOnly: el.readOnly, + value: el.valuePreview, + placeholder: el.placeholder, + tagName: el.tagName, + } + } + + // ── Fallback: data-agent-ref / aria-describedby (agent-browser CDP) ─ + const fallback = await wc.executeJavaScript(` (() => { const ref = ${JSON.stringify(targetRef)} const origin = ${JSON.stringify(originUrl)} const currentUrl = window.location.href - // Try to find the element by data-agent-ref attribute let el = document.querySelector('[data-agent-ref="' + ref + '"]') + || document.querySelector('[aria-describedby="' + ref + '"]') if (!el) { - // Fallback: the snapshot system may use aria attributes - el = document.querySelector('[aria-describedby="' + ref + '"]') - } - if (!el) { - // Last resort: look for elements whose computed aria label - // contains the ref — the ref @e5 may be embedded in an - // aria attribute generated by the accessibility mapper. return { found: false, reason: 'target_not_found', @@ -5734,25 +5960,20 @@ ipcMain.handle('hermes:browser:verify-action-target', async (_event, payload) => } } - // ── Read element metadata (purely read-only) ────────────────── const rect = el.getBoundingClientRect() const style = window.getComputedStyle(el) - const tagName = el.tagName + const tagName = el.tagName.toLowerCase() const textContent = (el.textContent || '').trim().slice(0, 200) const id = el.id || null const name = el.getAttribute('name') || null - const inputType = (tagName === 'INPUT' || tagName === 'TEXTAREA' || tagName === 'SELECT') + const inputType = (tagName === 'input' || tagName === 'textarea' || tagName === 'select') ? (el.getAttribute('type') || 'text') : null const ariaLabel = el.getAttribute('aria-label') || null const value = (el.value !== undefined && el.value !== null) ? String(el.value).slice(0, 200) : null const placeholder = el.getAttribute('placeholder') || null - - // Visibility checks const visible = ( - rect.width > 0 && - rect.height > 0 && - style.visibility !== 'hidden' && - style.display !== 'none' && + rect.width > 0 && rect.height > 0 && + style.visibility !== 'hidden' && style.display !== 'none' && parseFloat(style.opacity) > 0 ) const disabled = ( @@ -5772,7 +5993,7 @@ ipcMain.handle('hermes:browser:verify-action-target', async (_event, payload) => currentUrl: currentUrl, urlMatchesOrigin: currentUrl === origin, elementFingerprint: { - tagName: tagName.toLowerCase(), + tagName: tagName, textContent: textContent, id: id, name: name, @@ -5786,12 +6007,12 @@ ipcMain.handle('hermes:browser:verify-action-target', async (_event, payload) => readOnly: readOnly, value: value, placeholder: placeholder, - tagName: tagName.toLowerCase(), + tagName: tagName, } })() `) - return result + return fallback } catch (error) { return { found: false, @@ -5803,6 +6024,246 @@ ipcMain.handle('hermes:browser:verify-action-target', async (_event, payload) => } }) +// ── Phase 2F-B1: Execute click (read-only verify + input event) ───────── +// +// Executes a real mouse click on a verified interactive element. +// Before clicking, it RE-RUNS the same interactive candidate enumeration +// used by verify-action-target and re-checks ALL safety conditions. +// The verification is NOT trust-the-renderer — main process re-verifies +// independently. +// +// Click method: webContents.sendInputEvent({ type: 'mouseDown'/'mouseUp' }) +// at the element's boundingBox center. We do NOT use el.click() because: +// 1. sendInputEvent simulates a real user click (mouseDown + mouseUp). +// 2. el.click() bypasses the event pipeline — it fires the click handler +// directly, which can behave differently from a real user click +// (e.g. popup blockers, focus handling, trusted-event checks). +// 3. sendInputEvent is what Electron's own uses for +// programmatic clicks; it's the most faithful simulation. +// +// No focus(), no dispatchEvent(), no set value, no eval. + +ipcMain.handle('hermes:browser:execute-click', async (_event, payload) => { + try { + const targetRef = String(payload?.targetRef || '').trim() + const originUrl = String(payload?.originUrl || '').trim() + const expectedFingerprint = payload?.expectedFingerprint || null + + // ── Validate targetRef shape ─────────────────────────────────────── + if (!/^@e\d+$/.test(targetRef)) { + return { + ok: false, + reason: 'invalid_target_ref', + verification: { refValid: false, invalidationReason: 'invalid_target_ref' }, + } + } + + const targetIndex = Number.parseInt(targetRef.slice(2), 10) - 1 + if (targetIndex < 0 || !Number.isFinite(targetIndex)) { + return { + ok: false, + reason: 'invalid_target_ref', + verification: { refValid: false, invalidationReason: 'invalid_target_ref' }, + } + } + + const view = getBrowserView() + const wc = view.webContents + + // ── Re-enumerate interactive candidates (same script as verify) ──── + const snapshot = await wc.executeJavaScript(getInteractiveEnumerationScript()) + const candidates = snapshot?.elements || [] + const currentUrl = snapshot?.currentUrl || wc.getURL() + + // ── Check target exists ──────────────────────────────────────────── + if (targetIndex >= candidates.length) { + return { + ok: false, + reason: 'target_not_found', + currentUrl, + verification: { + refValid: false, + invalidationReason: 'target_not_found', + currentUrl, + }, + } + } + + const el = candidates[targetIndex] + + // ── Verify targetRef matches ─────────────────────────────────────── + if (el.ref !== targetRef) { + return { + ok: false, + reason: 'ref_mismatch', + currentUrl, + verification: { + refValid: false, + invalidationReason: 'ref_mismatch', + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } + + // ── Verify URL ───────────────────────────────────────────────────── + if (currentUrl !== originUrl) { + return { + ok: false, + reason: 'origin_url_mismatch', + currentUrl, + verification: { + refValid: false, + invalidationReason: 'origin_url_mismatch', + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } + + // ── Verify fingerprint ───────────────────────────────────────────── + // Mirrors compareElementFingerprint() in desktop-visible-provider.ts. + // Uses !== undefined guards (NOT falsy checks) so empty strings and + // null are compared rather than skipped. + if (expectedFingerprint) { + const fp = expectedFingerprint + const af = el.fingerprint + let fpMismatch = false + + if (fp.tagName !== undefined && fp.tagName !== null + && String(fp.tagName).toLowerCase() !== (af.tagName || '').toLowerCase()) {fpMismatch = true} + if (!fpMismatch && fp.textContent !== undefined && fp.textContent !== null + && String(fp.textContent) !== String(af.textContent || '')) {fpMismatch = true} + if (!fpMismatch && fp.id !== undefined && fp.id !== null + && fp.id !== af.id) {fpMismatch = true} + if (!fpMismatch && fp.name !== undefined && fp.name !== null + && fp.name !== af.name) {fpMismatch = true} + if (!fpMismatch && fp.inputType !== undefined && fp.inputType !== null + && fp.inputType !== af.inputType) {fpMismatch = true} + if (!fpMismatch && fp.ariaLabel !== undefined && fp.ariaLabel !== null + && fp.ariaLabel !== af.ariaLabel) {fpMismatch = true} + + if (fpMismatch) { + return { + ok: false, + reason: 'fingerprint_mismatch', + currentUrl, + verification: { + refValid: false, + invalidationReason: 'fingerprint_mismatch', + currentUrl, + currentFingerprint: af, + }, + } + } + } + + // ── Verify visible ───────────────────────────────────────────────── + if (!el.visible) { + return { + ok: false, + reason: 'target_not_visible', + currentUrl, + verification: { + refValid: false, + invalidationReason: 'target_not_visible', + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } + + // ── Verify not disabled ──────────────────────────────────────────── + if (el.disabled) { + return { + ok: false, + reason: 'target_disabled', + currentUrl, + verification: { + refValid: false, + invalidationReason: 'target_disabled', + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } + + // ── Semantic safety guard: reject high-risk targets ──────────────── + if (el.highRisk) { + const blockedRoles = ['submit_button', 'reset_button', 'file_input', 'password_input', 'external_link'] + const reason = blockedRoles.includes(el.semanticRole) + ? `Click rejected: target has semanticRole "${el.semanticRole}" which is blocked for safety.` + : `Click rejected: target "${el.textContent || el.tagName}" matches destructive keyword filter.` + return { + ok: false, + reason, + currentUrl, + verification: { + refValid: false, + invalidationReason: 'semantic_guard_blocked', + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } + + // ── Verify boundingBox ───────────────────────────────────────────── + const bb = el.boundingBox + if (!bb || bb.w <= 0 || bb.h <= 0) { + return { + ok: false, + reason: 'target_not_visible', + currentUrl, + verification: { + refValid: false, + invalidationReason: 'target_not_visible', + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } + + // ── Execute click via sendInputEvent ─────────────────────────────── + const clickX = bb.x + Math.floor(bb.w / 2) + const clickY = bb.y + Math.floor(bb.h / 2) + + try { + wc.sendInputEvent({ type: 'mouseDown', x: clickX, y: clickY, button: 'left', clickCount: 1 }) + wc.sendInputEvent({ type: 'mouseUp', x: clickX, y: clickY, button: 'left', clickCount: 1 }) + } catch (inputError) { + return { + ok: false, + reason: 'input_event_failed', + currentUrl, + detail: inputError?.message || String(inputError), + verification: { + refValid: true, + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } + + return { + ok: true, + currentUrl, + clickedAt: { x: clickX, y: clickY }, + verification: { + refValid: true, + currentUrl, + currentFingerprint: el.fingerprint, + }, + } + } catch (error) { + rememberLog(`[browser] execute-click IPC error: ${error?.message || String(error)}`) + return { + ok: false, + reason: 'ipc_error', + detail: error?.message || String(error), + } + } +}) + ipcMain.handle('hermes:updates:check', async () => checkUpdates().catch(error => ({ supported: true, diff --git a/apps/desktop/electron/preload.cjs b/apps/desktop/electron/preload.cjs index 07091258f0a8..2e27442d9fa6 100644 --- a/apps/desktop/electron/preload.cjs +++ b/apps/desktop/electron/preload.cjs @@ -125,6 +125,8 @@ contextBridge.exposeInMainWorld('hermesDesktop', { getDomSummary: () => ipcRenderer.invoke('hermes:browser:get-dom-summary'), getScreenshot: () => ipcRenderer.invoke('hermes:browser:get-screenshot'), getSelectedText: () => ipcRenderer.invoke('hermes:browser:get-selected-text'), + getInteractiveSnapshot: () => ipcRenderer.invoke('hermes:browser:get-interactive-snapshot'), + executeClick: payload => ipcRenderer.invoke('hermes:browser:execute-click', payload), verifyActionTarget: payload => ipcRenderer.invoke('hermes:browser:verify-action-target', payload), navigate: payload => ipcRenderer.invoke('hermes:browser:navigate', payload), reload: () => ipcRenderer.invoke('hermes:browser:reload', { source: 'user' }), diff --git a/apps/desktop/src/app/browser-runtime/action-gateway-ui.tsx b/apps/desktop/src/app/browser-runtime/action-gateway-ui.tsx index 136e0e3a7934..05eb4817de58 100644 --- a/apps/desktop/src/app/browser-runtime/action-gateway-ui.tsx +++ b/apps/desktop/src/app/browser-runtime/action-gateway-ui.tsx @@ -52,6 +52,7 @@ import { } from './action-gateway' import { type DesktopBrowserBridge, + executeDesktopClick, getDesktopSnapshot, VERIFICATION_FAILURE_REASONS, verifyDesktopActionTarget, @@ -147,8 +148,59 @@ export function BrowserActionGateway({ desktopBridge }: { desktopBridge?: Deskto } } - // ── Click/type — pre-action verification (Phase 2F-A) ──────────── - if (actionType === 'click' || actionType === 'type') { + // ── Click — real execution (Phase 2F-B1) ───────────────────────── + if (actionType === 'click') { + if (!desktopBridge) { + return { + status: 'failed' as const, + error: 'Desktop browser bridge is unavailable.', + } + } + + if (!request.safetyContext) { + return { + status: 'failed' as const, + error: 'Missing safety context — cannot execute click.', + } + } + + const result = await executeDesktopClick(desktopBridge, request.safetyContext) + + if (!result.ok) { + return { + status: 'failed' as const, + error: `Click failed: ${result.reason || 'unknown'}.`, + postActionSnapshot: result.postActionSnapshot, + preActionVerification: result.verification + ? { + verifiedAt: new Date().toISOString(), + currentUrl: result.currentUrl || '', + refValid: result.verification.refValid, + invalidationReason: result.verification.invalidationReason, + currentFingerprint: result.verification.currentFingerprint as PreActionVerification['currentFingerprint'], + snapshot: result.postActionSnapshot || ({} as PreActionVerification['snapshot']), + } + : undefined, + } + } + + return { + status: 'executed' as const, + postActionSnapshot: result.postActionSnapshot, + preActionVerification: result.verification + ? { + verifiedAt: new Date().toISOString(), + currentUrl: result.currentUrl || '', + refValid: result.verification.refValid, + currentFingerprint: result.verification.currentFingerprint as PreActionVerification['currentFingerprint'], + snapshot: result.postActionSnapshot || ({} as PreActionVerification['snapshot']), + } + : undefined, + } + } + + // ── Type — pre-action verification only (Phase 2F-A) ───────────── + if (actionType === 'type') { let preActionVerification: PreActionVerification | undefined if (request.safetyContext && desktopBridge) { @@ -172,13 +224,11 @@ export function BrowserActionGateway({ desktopBridge }: { desktopBridge?: Deskto } } - // Phase 2F-A does NOT execute real click/type. - // The verification result is attached to the log for the user to inspect. return { status: 'failed' as const, - error: `"${actionType}" execution is not yet implemented (Phase 2F-A). ` + error: '"type" execution is not yet implemented (Phase 2F-A). ' + (preActionVerification?.refValid - ? 'Pre-action verification passed — target element found and matches safety context. Ready for Phase 2F-B executor.' + ? 'Pre-action verification passed. Ready for Phase 2F-B2 executor.' : `Pre-action verification failed: ${preActionVerification?.invalidationReason || 'unknown'}. `), preActionVerification, } diff --git a/apps/desktop/src/app/browser-runtime/action-gateway.test.ts b/apps/desktop/src/app/browser-runtime/action-gateway.test.ts index 88c93f44ecea..cf89927585a1 100644 --- a/apps/desktop/src/app/browser-runtime/action-gateway.test.ts +++ b/apps/desktop/src/app/browser-runtime/action-gateway.test.ts @@ -452,8 +452,8 @@ describe('Phase 2E — action classification sets', () => { expect(PERMANENTLY_DENIED_ACTIONS.has('navigate')).toBe(false) }) - it('AWAITING_SAFETY_ACTIONS contains click and type', () => { - expect(AWAITING_SAFETY_ACTIONS.has('click')).toBe(true) + it('AWAITING_SAFETY_ACTIONS contains only type', () => { + expect(AWAITING_SAFETY_ACTIONS.has('type')).toBe(true) expect(AWAITING_SAFETY_ACTIONS.has('type')).toBe(true) }) @@ -462,9 +462,8 @@ describe('Phase 2E — action classification sets', () => { expect(AWAITING_SAFETY_ACTIONS.has('eval')).toBe(false) }) - it('EXECUTABLE_ACTIONS only contains navigate', () => { - expect(EXECUTABLE_ACTIONS.has('navigate')).toBe(true) - expect(EXECUTABLE_ACTIONS.has('click')).toBe(false) + it('EXECUTABLE_ACTIONS contains click and navigate', () => { + expect(EXECUTABLE_ACTIONS.has('click')).toBe(true) expect(EXECUTABLE_ACTIONS.has('type')).toBe(false) expect(EXECUTABLE_ACTIONS.has('eval')).toBe(false) expect(EXECUTABLE_ACTIONS.has('snapshot')).toBe(false) @@ -476,8 +475,8 @@ describe('Phase 2E — isActionExecutable', () => { expect(isActionExecutable('navigate')).toBe(true) }) - it('returns false for click, type, eval, scroll, press_key', () => { - expect(isActionExecutable('click')).toBe(false) + it('returns true for click, false for type, eval, scroll, press_key', () => { + expect(isActionExecutable('click')).toBe(true) expect(isActionExecutable('type')).toBe(false) expect(isActionExecutable('eval')).toBe(false) expect(isActionExecutable('scroll')).toBe(false) @@ -507,7 +506,7 @@ describe('Phase 2E — getBlockedActionReason', () => { }) it('returns a reason for awaiting-safety actions', () => { - const reason = getBlockedActionReason('click') + const reason = getBlockedActionReason('type') expect(reason).toBeTruthy() expect(reason).toContain('Phase 2E') expect(reason).toContain('Phase 2F') @@ -584,8 +583,8 @@ describe('Phase 2E — proposeAction accepts safetyContext', () => { }) describe('Phase 2E — click/type non-execution contract', () => { - it('click is NOT in EXECUTABLE_ACTIONS', () => { - expect(EXECUTABLE_ACTIONS.has('click')).toBe(false) + it('click IS in EXECUTABLE_ACTIONS', () => { + expect(EXECUTABLE_ACTIONS.has('click')).toBe(true) }) it('type is NOT in EXECUTABLE_ACTIONS', () => { @@ -597,8 +596,8 @@ describe('Phase 2E — click/type non-execution contract', () => { expect(AWAITING_SAFETY_ACTIONS.has('eval')).toBe(false) }) - it('navigate is the only executable action', () => { - expect([...EXECUTABLE_ACTIONS]).toEqual(['navigate']) + it('click and navigate are the only executable actions', () => { + expect([...EXECUTABLE_ACTIONS].sort()).toEqual(['click', 'navigate']) }) }) @@ -640,17 +639,17 @@ describe('Phase 2F-A — read-only actions produce real data', () => { }) describe('Phase 2F-A — EXECUTABLE_ACTIONS still only contains navigate', () => { - it('only navigate is directly executable', () => { - expect([...EXECUTABLE_ACTIONS].sort()).toEqual(['navigate']) + it('click and navigate are directly executable', () => { + expect([...EXECUTABLE_ACTIONS].sort()).toEqual(['click', 'navigate']) }) - it('click and type are STILL not executable', () => { - expect(EXECUTABLE_ACTIONS.has('click')).toBe(false) - expect(EXECUTABLE_ACTIONS.has('type')).toBe(false) + it('type is STILL not executable', () => { + expect(EXECUTABLE_ACTIONS.has('click')).toBe(true) + expect(EXECUTABLE_ACTIONS.has('type')).toBe(false); expect(EXECUTABLE_ACTIONS.has('click')).toBe(true) }) - it('click and type are STILL in AWAITING_SAFETY_ACTIONS', () => { - expect(AWAITING_SAFETY_ACTIONS.has('click')).toBe(true) + it('type is STILL in AWAITING_SAFETY_ACTIONS', () => { + expect(AWAITING_SAFETY_ACTIONS.has('type')).toBe(true) expect(AWAITING_SAFETY_ACTIONS.has('type')).toBe(true) }) }) @@ -688,3 +687,43 @@ describe('Phase 2F-A — proposeAction with safetyContext for verification', () expect(pending?.safetyContext?.elementFingerprint?.tagName).toBe('BUTTON') }) }) + +// ═══════════════════════════════════════════════════════════════════════ +// Phase 2F-B1 — Real click execution contract +// ═══════════════════════════════════════════════════════════════════════ + +describe('Phase 2F-B1 — click execution contract', () => { + it('EXECUTABLE_ACTIONS contains click', () => { + expect(EXECUTABLE_ACTIONS.has('click')).toBe(true) + }) + + it('EXECUTABLE_ACTIONS contains navigate', () => { + expect(EXECUTABLE_ACTIONS.has('navigate')).toBe(true) + }) + + it('EXECUTABLE_ACTIONS does NOT contain type', () => { + expect(EXECUTABLE_ACTIONS.has('type')).toBe(false) + }) + + it('AWAITING_SAFETY_ACTIONS only contains type', () => { + expect(AWAITING_SAFETY_ACTIONS.has('type')).toBe(true) + expect(AWAITING_SAFETY_ACTIONS.has('click')).toBe(false) + }) + + it('PERMANENTLY_DENIED_ACTIONS unchanged', () => { + expect(PERMANENTLY_DENIED_ACTIONS.has('eval')).toBe(true) + expect(PERMANENTLY_DENIED_ACTIONS.has('press_key')).toBe(true) + expect(PERMANENTLY_DENIED_ACTIONS.has('scroll')).toBe(true) + expect(PERMANENTLY_DENIED_ACTIONS.has('click')).toBe(false) + }) + + it('getBlockedActionReason returns null for click (executable)', () => { + expect(getBlockedActionReason('click')).toBeNull() + }) + + it('getBlockedActionReason returns a reason for type (awaiting)', () => { + const reason = getBlockedActionReason('type') + expect(reason).toBeTruthy() + expect(reason).toContain('type') + }) +}) diff --git a/apps/desktop/src/app/browser-runtime/action-gateway.ts b/apps/desktop/src/app/browser-runtime/action-gateway.ts index b0e3ad3f20f8..d5e47b734733 100644 --- a/apps/desktop/src/app/browser-runtime/action-gateway.ts +++ b/apps/desktop/src/app/browser-runtime/action-gateway.ts @@ -243,7 +243,6 @@ export const PERMANENTLY_DENIED_ACTIONS: ReadonlySet = new Set([ * @see docs/architecture/desktop-browser-agent-action-safety.md §7.3 */ export const AWAITING_SAFETY_ACTIONS: ReadonlySet = new Set([ - 'click', 'type', ]) @@ -252,6 +251,7 @@ export const AWAITING_SAFETY_ACTIONS: ReadonlySet = new Set([ * approval (Phase 2C). */ export const EXECUTABLE_ACTIONS: ReadonlySet = new Set([ + 'click', 'navigate', ]) diff --git a/apps/desktop/src/app/browser-runtime/desktop-visible-provider.test.ts b/apps/desktop/src/app/browser-runtime/desktop-visible-provider.test.ts index bdfba9fe958c..50c9ca8c085b 100644 --- a/apps/desktop/src/app/browser-runtime/desktop-visible-provider.test.ts +++ b/apps/desktop/src/app/browser-runtime/desktop-visible-provider.test.ts @@ -8,12 +8,15 @@ import { describe, expect, it } from 'vitest' import { + buildSafetyContextFromElement, checkDesktopPermission, DESKTOP_VISIBLE_CAPABILITIES, DESKTOP_VISIBLE_DEFAULT_POLICIES, DESKTOP_VISIBLE_DESCRIPTOR, DESKTOP_VISIBLE_ID, type DesktopBrowserBridge, + executeDesktopClick, + getDesktopInteractiveSnapshot, getDesktopSnapshot, isDesktopActionAllowed, isDesktopActionApprovalRequired, @@ -150,8 +153,8 @@ describe('DESKTOP_VISIBLE_DESCRIPTOR', () => { describe('policy invariants', () => { describe('agent actions', () => { - it('interactive agent actions (click, type, eval, press_key, scroll) are DENIED', () => { - const interactiveActions = ['click', 'type', 'eval', 'press_key', 'scroll'] + it('interactive agent actions (type, eval, press_key, scroll) are DENIED', () => { + const interactiveActions = ['type', 'eval', 'press_key', 'scroll'] for (const action of interactiveActions) { expect(checkDesktopPermission(action, 'agent')).toBe('deny') @@ -197,8 +200,8 @@ describe('policy invariants', () => { expect(checkDesktopPermission('console', 'system')).toBe('allow') }) - it('system interactive actions (click, type, eval) are denied', () => { - expect(checkDesktopPermission('click', 'system')).toBe('deny') + it('system interactive actions (type, eval) are denied (click is approval_required)', () => { + expect(checkDesktopPermission('click', 'system')).toBe('approval_required') expect(checkDesktopPermission('type', 'system')).toBe('deny') expect(checkDesktopPermission('eval', 'system')).toBe('deny') }) @@ -224,8 +227,8 @@ describe('checkDesktopPermission', () => { expect(checkDesktopPermission('back', 'user')).toBe('allow') }) - it('agent cannot click, type, or eval', () => { - expect(checkDesktopPermission('click', 'agent')).toBe('deny') + it('agent cannot type or eval (click is approval_required)', () => { + expect(checkDesktopPermission('click', 'agent')).toBe('approval_required') expect(checkDesktopPermission('type', 'agent')).toBe('deny') expect(checkDesktopPermission('eval', 'agent')).toBe('deny') }) @@ -237,10 +240,10 @@ describe('checkDesktopPermission', () => { expect(checkDesktopPermission('console', 'agent')).toBe('allow') }) - it('system can read but navigate requires approval', () => { + it('system can read, click requires approval, navigate requires approval', () => { expect(checkDesktopPermission('snapshot', 'system')).toBe('allow') expect(checkDesktopPermission('navigate', 'system')).toBe('approval_required') - expect(checkDesktopPermission('click', 'system')).toBe('deny') + expect(checkDesktopPermission('click', 'system')).toBe('approval_required') }) it('returns deny for unknown agent action, allow for system (catch-all)', () => { @@ -511,11 +514,390 @@ describe('Phase 2D — permanent deny policies', () => { }) describe('click and type are denied (awaiting Phase 2F safety implementation)', () => { - it('agent click → deny (will become approval_required in Phase 2F)', () => { - expect(checkDesktopPermission('click', 'agent')).toBe('deny') + it('agent click → approval_required (Phase 2F-B1)', () => { + expect(checkDesktopPermission('click', 'agent')).toBe('approval_required') }) it('agent type → deny (will become approval_required in Phase 2F)', () => { expect(checkDesktopPermission('type', 'agent')).toBe('deny') }) }) }) + +// ═══════════════════════════════════════════════════════════════════════ +// 9. Phase 2F-A2 — Stable ref mapping contract +// ═══════════════════════════════════════════════════════════════════════ + +describe('Phase 2F-A2 — stable ref mapping', () => { + it('buildSafetyContextFromElement produces valid safetyContext with targetRef', () => { + + const el = { + ref: '@e1', + tagName: 'button', + role: null, + semanticRole: 'submit_button' as const, + highRisk: true, + isDestructive: false, + isMediumRisk: false, + href: null, + textContent: 'Submit PR', + ariaLabel: null, + id: 'submit-btn', + name: null, + inputType: 'submit', + placeholder: null, + valuePreview: null, + boundingBox: { x: 100, y: 200, w: 120, h: 36 }, + visible: true, + disabled: false, + readOnly: false, + fingerprint: { + tagName: 'button', + textContent: 'Submit PR', + id: 'submit-btn', + name: null, + inputType: 'submit', + ariaLabel: null, + rect: { x: 100, y: 200, w: 120, h: 36 }, + }, + } + + const ctx = buildSafetyContextFromElement( + el, + 'https://example.com', + 'Example Page', + 'click', + ) + + expect(ctx.targetRef).toBe('@e1') + expect(ctx.originUrl).toBe('https://example.com') + expect(ctx.targetDescription).toContain('button') + expect(ctx.targetDescription).toContain('Submit PR') + expect(ctx.riskLevel).toBe('medium') + expect(ctx.elementFingerprint).toEqual(el.fingerprint) + expect(ctx.typeText).toBeUndefined() // click, not type + }) + + it('buildSafetyContextFromElement includes typeText for type action', () => { + + const el = { + ref: '@e3', + tagName: 'input', + role: null, + semanticRole: 'text_input' as const, + highRisk: false, + isDestructive: false, + isMediumRisk: false, + href: null, + textContent: '', + ariaLabel: null, + id: null, + name: 'q', + inputType: 'text', + placeholder: 'Search…', + valuePreview: 'current query', + boundingBox: { x: 100, y: 200, w: 200, h: 28 }, + visible: true, + disabled: false, + readOnly: false, + fingerprint: { + tagName: 'input', + textContent: '', + id: null, + name: 'q', + inputType: 'text', + ariaLabel: null, + rect: { x: 100, y: 200, w: 200, h: 28 }, + }, + } + + const ctx = buildSafetyContextFromElement( + el, + 'https://example.com', + 'Example Page', + 'type', + ) + + expect(ctx.targetRef).toBe('@e3') + expect(ctx.typeText).toBe('current query') + }) +}) + +describe('Phase 2F-A2 — getDesktopInteractiveSnapshot', () => { + it('returns null when bridge has no getInteractiveSnapshot', async () => { + + const bridge = fakeBridge() + const result = await getDesktopInteractiveSnapshot(bridge) + expect(result).toBeNull() + }) + + it('returns null when IPC returns ok:false', async () => { + + const bridge = { + ...fakeBridge(), + getInteractiveSnapshot: async () => ({ + ok: false, + capturedAt: new Date().toISOString(), + currentUrl: '', + elements: [], + error: 'not mounted', + }), + } + + const result = await getDesktopInteractiveSnapshot(bridge) + expect(result).toBeNull() + }) + + it('returns snapshot when IPC succeeds', async () => { + + const el = { + ref: '@e1', + tagName: 'button', + role: null, + semanticRole: 'button' as const, + highRisk: false, + isDestructive: false, + isMediumRisk: false, + href: null, + textContent: 'Click me', + ariaLabel: null, + id: null, + name: null, + inputType: null, + placeholder: null, + valuePreview: null, + boundingBox: { x: 0, y: 0, w: 100, h: 40 }, + visible: true, + disabled: false, + readOnly: false, + fingerprint: { + tagName: 'button', + textContent: 'Click me', + id: null, + name: null, + inputType: null, + ariaLabel: null, + rect: { x: 0, y: 0, w: 100, h: 40 }, + }, + } + + const bridge = { + ...fakeBridge(), + getInteractiveSnapshot: async () => ({ + ok: true, + capturedAt: new Date().toISOString(), + currentUrl: 'https://example.com', + elements: [el], + }), + } + + const result = await getDesktopInteractiveSnapshot(bridge) + expect(result).not.toBeNull() + expect(result!.elements).toHaveLength(1) + expect(result!.elements[0].ref).toBe('@e1') + expect(result!.elements[0].tagName).toBe('button') + }) +}) + +// ═══════════════════════════════════════════════════════════════════════ +// 10. Phase 2F-B — Semantic click safety guards +// ═══════════════════════════════════════════════════════════════════════ + +describe('Phase 2F-B — semantic click guards', () => { + const mockSnapshot = (overrides: Record = {}) => ({ + ok: true as const, + capturedAt: new Date().toISOString(), + currentUrl: 'https://example.com', + elements: [{ + ref: '@e1', + tagName: 'button', + role: null, + semanticRole: 'button', + highRisk: false, + isDestructive: false, + isMediumRisk: false, + href: null, + textContent: 'Click me', + ariaLabel: null, + id: null, + name: null, + inputType: null, + placeholder: null, + valuePreview: null, + boundingBox: { x: 0, y: 0, w: 100, h: 40 }, + visible: true, + disabled: false, + readOnly: false, + fingerprint: { + tagName: 'button', + textContent: 'Click me', + id: null, + name: null, + inputType: null, + ariaLabel: null, + rect: { x: 0, y: 0, w: 100, h: 40 }, + }, + ...overrides, + }], + }) + + it('executeClick rejects submit_button', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async (payload: { targetRef: string }) => { + // Simulate IPC returning blocked for highRisk submit_button + return { + ok: false, + reason: `Click rejected: target has semanticRole "submit_button"`, + currentUrl: 'https://example.com', + verification: { refValid: false, invalidationReason: 'semantic_guard_blocked' }, + } + }, + } + + const result = await executeDesktopClick(bridge, { + targetRef: '@e1', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + expect(result.reason).toContain('submit_button') + }) + + it('executeClick rejects destructive labels', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async () => ({ + ok: false, + reason: 'Click rejected: target "Delete account" matches destructive keyword filter.', + currentUrl: 'https://example.com', + verification: { refValid: false, invalidationReason: 'semantic_guard_blocked' }, + }), + } + + const result = await executeDesktopClick(bridge, { + targetRef: '@e2', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + expect(result.reason).toContain('destructive') + }) + + it('executeClick rejects file_input', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async () => ({ + ok: false, + reason: 'Click rejected: target has semanticRole "file_input" which is blocked for safety.', + currentUrl: 'https://example.com', + verification: { refValid: false, invalidationReason: 'semantic_guard_blocked' }, + }), + } + + const result = await executeDesktopClick(bridge, { + targetRef: '@e3', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + expect(result.reason).toContain('file_input') + }) + + it('executeClick rejects external_link', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async () => ({ + ok: false, + reason: 'Click rejected: target has semanticRole "external_link" which is blocked for safety.', + currentUrl: 'https://example.com', + verification: { refValid: false, invalidationReason: 'semantic_guard_blocked' }, + }), + } + + const result = await executeDesktopClick(bridge, { + targetRef: '@e4', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + expect(result.reason).toContain('external_link') + }) + + it('executeClick still rejects stale targetRef', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async () => ({ + ok: false, + reason: 'target_not_found', + currentUrl: 'https://example.com', + verification: { refValid: false, invalidationReason: 'target_not_found' }, + }), + } + + const result = await executeDesktopClick(bridge, { + targetRef: '@e99', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + expect(result.reason).toContain('target_not_found') + }) + + it('executeClick rejects invalid @e ref', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async () => ({ + ok: false, + reason: 'invalid_target_ref', + verification: { refValid: false, invalidationReason: 'invalid_target_ref' }, + }), + } + + const result = await executeDesktopClick(bridge, { + targetRef: 'bad-ref', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + }) + + it('executeClick rejects password_input', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async () => ({ + ok: false, + reason: 'Click rejected: target has semanticRole "password_input" which is blocked for safety.', + currentUrl: 'https://example.com', + verification: { refValid: false, invalidationReason: 'semantic_guard_blocked' }, + }), + } + + const result = await executeDesktopClick(bridge, { + targetRef: '@e5', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + expect(result.reason).toContain('password_input') + }) + + it('executeClick rejects reset_button', async () => { + const bridge = { + ...fakeBridge(), + executeClick: async () => ({ + ok: false, + reason: 'Click rejected: target has semanticRole "reset_button" which is blocked for safety.', + currentUrl: 'https://example.com', + verification: { refValid: false, invalidationReason: 'semantic_guard_blocked' }, + }), + } + + const result = await executeDesktopClick(bridge, { + targetRef: '@e6', + originUrl: 'https://example.com', + }) + + expect(result.ok).toBe(false) + expect(result.reason).toContain('reset_button') + }) +}) diff --git a/apps/desktop/src/app/browser-runtime/desktop-visible-provider.ts b/apps/desktop/src/app/browser-runtime/desktop-visible-provider.ts index 63bed00cd0e9..7cd35e027901 100644 --- a/apps/desktop/src/app/browser-runtime/desktop-visible-provider.ts +++ b/apps/desktop/src/app/browser-runtime/desktop-visible-provider.ts @@ -17,6 +17,7 @@ */ import type { + BrowserActionSafetyContext, BrowserActiveTabContext, BrowserCapability, BrowserConsoleSnapshot, @@ -50,6 +51,10 @@ export interface DesktopBrowserBridge { getDomSummary(): Promise getScreenshot(): Promise getSelectedText(): Promise + /** Phase 2F-A2: Enumerate interactive elements. */ + getInteractiveSnapshot?(): Promise + /** Phase 2F-B1: Execute real click (re-verifies before clicking). */ + executeClick?(payload: { targetRef: string; originUrl: string; expectedFingerprint?: Record }): Promise /** Phase 2F-A: Read-only target resolution IPC. */ verifyActionTarget?(payload: { targetRef: string; originUrl: string }): Promise } @@ -135,8 +140,9 @@ export const DESKTOP_VISIBLE_DEFAULT_POLICIES: BrowserPermissionPolicy[] = [ { provider: 'desktop-visible', action: 'eval', actor: 'agent', decision: 'deny' }, { provider: 'desktop-visible', action: 'press_key', actor: 'agent', decision: 'deny' }, { provider: 'desktop-visible', action: 'scroll', actor: 'agent', decision: 'deny' }, - // Denied until Phase 2F safety implementation (will become approval_required): - { provider: 'desktop-visible', action: 'click', actor: 'agent', decision: 'deny' }, + // click — executable after approval (Phase 2F-B1): + { provider: 'desktop-visible', action: 'click', actor: 'agent', decision: 'approval_required' }, + // type — denied until Phase 2F-B2: { provider: 'desktop-visible', action: 'type', actor: 'agent', decision: 'deny' }, // Navigation — user must approve: { provider: 'desktop-visible', action: 'navigate', actor: 'agent', decision: 'approval_required' }, @@ -157,7 +163,7 @@ export const DESKTOP_VISIBLE_DEFAULT_POLICIES: BrowserPermissionPolicy[] = [ { provider: 'desktop-visible', action: 'eval', actor: 'system', decision: 'deny' }, { provider: 'desktop-visible', action: 'press_key', actor: 'system', decision: 'deny' }, { provider: 'desktop-visible', action: 'scroll', actor: 'system', decision: 'deny' }, - { provider: 'desktop-visible', action: 'click', actor: 'system', decision: 'deny' }, + { provider: 'desktop-visible', action: 'click', actor: 'system', decision: 'approval_required' }, { provider: 'desktop-visible', action: 'type', actor: 'system', decision: 'deny' }, { provider: 'desktop-visible', action: 'navigate', actor: 'system', decision: 'approval_required' }, { provider: 'desktop-visible', action: 'back', actor: 'system', decision: 'approval_required' }, @@ -409,6 +415,49 @@ export async function getDesktopSnapshot( // 5. Phase 2F-A: Read-only target verification // ═══════════════════════════════════════════════════════════════════════════ +/** + * Shape returned by the `hermes:browser:get-interactive-snapshot` IPC. + * Mirrors `DesktopInteractiveSnapshotResult` in global.d.ts. + */ +export interface DesktopInteractiveSnapshotResult { + ok: boolean + capturedAt: string + currentUrl: string + elements: DesktopInteractiveSnapshotElement[] + error?: string +} + +export interface DesktopInteractiveSnapshotElement { + ref: string + tagName: string + role: string | null + semanticRole: string + highRisk: boolean + isDestructive: boolean + isMediumRisk: boolean + textContent: string + ariaLabel: string | null + id: string | null + name: string | null + inputType: string | null + placeholder: string | null + valuePreview: string | null + href: string | null + boundingBox: { x: number; y: number; w: number; h: number } + visible: boolean + disabled: boolean + readOnly: boolean + fingerprint: { + tagName: string + textContent: string + id: string | null + name: string | null + inputType: string | null + ariaLabel: string | null + rect: { x: number; y: number; w: number; h: number } + } +} + /** * Shape returned by the `hermes:browser:verify-action-target` IPC. * Mirrors `DesktopVerifyTargetResult` in global.d.ts. @@ -437,6 +486,89 @@ export interface DesktopVerifyTargetResult { detail?: string } +/** + * Shape returned by the ``hermes:browser:execute-click`` IPC. + */ +export interface DesktopExecuteClickResult { + ok: boolean + reason?: string + detail?: string + currentUrl?: string + clickedAt?: { x: number; y: number } + verification?: { + refValid: boolean + invalidationReason?: string + currentUrl?: string + currentFingerprint?: { + tagName: string + textContent: string + id: string | null + name: string | null + inputType: string | null + ariaLabel: string | null + } + } +} + +/** + * Pure function: compare an expected element fingerprint against an actual + * one. Returns null when they match, or a mismatch reason string. + * + * IMPORTANT: uses ``!== undefined`` guards, NOT falsy checks — an empty + * string in the expected fingerprint MUST be compared, not skipped. + * Kept in sync with the equivalent logic in ``main.cjs`` execute-click. + */ +export function compareElementFingerprint( + expected: { + tagName?: string | null + textContent?: string | null + id?: string | null + name?: string | null + inputType?: string | null + ariaLabel?: string | null + }, + actual: { + tagName?: string | null + textContent?: string | null + id?: string | null + name?: string | null + inputType?: string | null + ariaLabel?: string | null + }, +): string | null { + if (expected.tagName !== undefined && expected.tagName !== null + && (actual.tagName == null || expected.tagName.toLowerCase() !== String(actual.tagName).toLowerCase())) { + return 'tagName' + } + + if (expected.textContent !== undefined && expected.textContent !== null + && expected.textContent !== String(actual.textContent ?? '')) { + return 'textContent' + } + + if (expected.id !== undefined && expected.id !== null + && expected.id !== actual.id) { + return 'id' + } + + if (expected.name !== undefined && expected.name !== null + && expected.name !== actual.name) { + return 'name' + } + + if (expected.inputType !== undefined && expected.inputType !== null + && expected.inputType !== actual.inputType) { + return 'inputType' + } + + if (expected.ariaLabel !== undefined && expected.ariaLabel !== null + && expected.ariaLabel !== actual.ariaLabel) { + return 'ariaLabel' + } + + return null +} + /** * Known verification failure reasons (readable constants). */ @@ -548,22 +680,18 @@ export async function verifyDesktopActionTarget( // ── Fingerprint mismatch ──────────────────────────────────────────── if (safetyContext.elementFingerprint && ipcResult.elementFingerprint) { - const expected = safetyContext.elementFingerprint - const actual = ipcResult.elementFingerprint - - if ( - expected.tagName.toLowerCase() !== actual.tagName.toLowerCase() - || expected.textContent !== actual.textContent - || expected.id !== actual.id - || expected.name !== actual.name - || expected.inputType !== actual.inputType - ) { + const mismatchField = compareElementFingerprint( + safetyContext.elementFingerprint, + ipcResult.elementFingerprint, + ) + + if (mismatchField) { return { verifiedAt, currentUrl: ipcResult.currentUrl, refValid: false, invalidationReason: VERIFICATION_FAILURE_REASONS.fingerprint_mismatch, - currentFingerprint: actual, + currentFingerprint: ipcResult.elementFingerprint, snapshot, } } @@ -603,7 +731,89 @@ export async function verifyDesktopActionTarget( } } -// ═══════════════════════════════════════════════════════════════════════ +// ═══════════════════════════════════════════════════════════════════════════ +// 6. Execute click (Phase 2F-B1) +// ═══════════════════════════════════════════════════════════════════════════ + +/** + * + * Calls `hermes:browser:execute-click` which re-runs the interactive + * candidate enumeration and checks URL match, fingerprint match, visibility, + * and disabled status *in the main process* before sending mouseDown/mouseUp. + * + * Does NOT execute type, eval, press_key, or scroll. + * + * @param bridge The desktop browser bridge. + * @param safetyContext The verified safety context (must have targetRef, originUrl, elementFingerprint). + * @returns Partial BrowserActionResult fields for the gateway. + */ +export async function executeDesktopClick( + bridge: DesktopBrowserBridge, + safetyContext: { + targetRef: string + originUrl: string + elementFingerprint?: { + tagName: string + textContent: string + id: string | null + name: string | null + inputType: string | null + ariaLabel: string | null + } + }, +): Promise<{ + ok: boolean + reason?: string + clickedAt?: { x: number; y: number } + currentUrl?: string + verification?: DesktopExecuteClickResult['verification'] + postActionSnapshot?: BrowserSnapshot +}> { + if (!bridge.executeClick) { + return { ok: false, reason: 'bridge_unavailable' } + } + + let result: DesktopExecuteClickResult + + try { + result = await bridge.executeClick({ + targetRef: safetyContext.targetRef, + originUrl: safetyContext.originUrl, + expectedFingerprint: safetyContext.elementFingerprint as Record | undefined, + }) + } catch (error) { + return { ok: false, reason: `executeClick IPC failed: ${error instanceof Error ? error.message : String(error)}` } + } + + if (!result.ok) { + return { + ok: false, + reason: result.reason || 'click_failed', + currentUrl: result.currentUrl, + verification: result.verification, + } + } + + // Capture post-click snapshot + let postActionSnapshot: BrowserSnapshot | undefined + + try { + postActionSnapshot = await getDesktopSnapshot(bridge) + } catch { + // best-effort + } + + return { + ok: true, + clickedAt: result.clickedAt, + currentUrl: result.currentUrl, + verification: result.verification, + postActionSnapshot, + } +} + +// ═══════════════════════════════════════════════════════════════════════════ +// 7. Availability check // 6. Availability check // ═══════════════════════════════════════════════════════════════════════ @@ -625,3 +835,63 @@ export async function isDesktopVisibleAvailable( return { available: false, reason: `isAvailable IPC failed: ${String(error)}` } } } + +// ═══════════════════════════════════════════════════════════════════════════ +// 6b. Phase 2F-A2: Interactive snapshot helpers +// ═══════════════════════════════════════════════════════════════════════════ + +/** + * Get the interactive element snapshot from the Desktop browser. + * + * Calls the `hermes:browser:get-interactive-snapshot` IPC to enumerate + * all interactive elements on the current page with stable `@eN` refs. + * Returns null when the bridge or IPC is unavailable. + */ +export async function getDesktopInteractiveSnapshot( + bridge: DesktopBrowserBridge, +): Promise { + if (!bridge.getInteractiveSnapshot) {return null} + + try { + const result = await bridge.getInteractiveSnapshot() + + if (!result.ok) {return null} + + return result + } catch { + return null + } +} + +/** + * Build a minimal BrowserActionSafetyContext from an interactive snapshot + * element, suitable for demo / testing. Uses the element's fingerprint + * and metadata to populate the safety fields. + */ +export function buildSafetyContextFromElement( + el: DesktopInteractiveSnapshotElement, + originUrl: string, + originTitle: string, + actionType: 'click' | 'type', +): BrowserActionSafetyContext { + const descriptionParts: string[] = [] + descriptionParts.push(el.tagName) + + if (el.inputType) {descriptionParts.push(`(type: ${el.inputType})`)} + + if (el.id) {descriptionParts.push(`#${el.id}`)} + + if (el.role) {descriptionParts.push(`role: ${el.role}`)} + + if (el.textContent) {descriptionParts.push(`"${el.textContent.slice(0, 40)}"`)} + + return { + originUrl, + originTitle, + targetDescription: descriptionParts.join(' '), + targetRef: el.ref, + typeText: actionType === 'type' ? (el.valuePreview || '') : undefined, + elementFingerprint: el.fingerprint, + riskLevel: 'medium', + } +} diff --git a/apps/desktop/src/app/browser-workspace.tsx b/apps/desktop/src/app/browser-workspace.tsx index 698319a8ca4f..f762d1f5e64e 100644 --- a/apps/desktop/src/app/browser-workspace.tsx +++ b/apps/desktop/src/app/browser-workspace.tsx @@ -14,6 +14,10 @@ import { cn } from '@/lib/utils' import { proposeAction } from './browser-runtime/action-gateway' import { BrowserActionGateway } from './browser-runtime/action-gateway-ui' +import { + buildSafetyContextFromElement, + getDesktopInteractiveSnapshot, +} from './browser-runtime/desktop-visible-provider' import type { BrowserActionSafetyContext } from './browser-runtime/types' import { WorkspaceLauncher } from './workspace-launcher' @@ -514,35 +518,69 @@ export function BrowserWorkspace() { 'desktop-visible', ) }} /> - { + { + if (!bridge) {return} + const snap = await getDesktopInteractiveSnapshot(bridge) + + if (!snap || snap.elements.length === 0) { + // Propose a dummy click so the user can see "no target" in the UI + proposeAction( + { type: 'click', ref: '@e1' }, + 'agent', 'demo_task', + 'No interactive elements found on this page.', + 'desktop-visible', + ) + + return + } + + // Use the first visible, non-disabled element + const el = snap.elements.find(e => e.visible && !e.disabled) || snap.elements[0] + + const ctx = buildSafetyContextFromElement( + el, snap.currentUrl, page.title || 'Desktop Page', 'click', + ) + proposeAction( - { type: 'click', ref: '@e5' }, + { type: 'click', ref: el.ref }, 'agent', 'demo_task', - 'Agent wants to click the Submit button on the PR form', + `Agent wants to click ${el.tagName} "${el.textContent.slice(0, 40)}"`, 'desktop-visible', - { - originUrl: page.url || 'https://github.com/gu/trendradar/pull/42', - originTitle: page.title || 'Pull Request #42', - targetDescription: "button 'Submit PR' (tag: button, type: submit) near heading 'Create Pull Request'", - targetRef: '@e5', - riskLevel: 'medium', - } satisfies BrowserActionSafetyContext, + ctx, ) }} /> - { + { + if (!bridge) {return} + const snap = await getDesktopInteractiveSnapshot(bridge) + + if (!snap) {return} + + // Find the first visible, non-disabled input/textarea + const input = snap.elements.find( + e => e.visible && !e.disabled && (e.tagName === 'input' || e.tagName === 'textarea'), + ) + + if (!input) { + proposeAction( + { type: 'type', ref: '@e1', text: '' }, + 'agent', 'demo_task', + 'No input/textarea element found on this page.', + 'desktop-visible', + ) + + return + } + + const ctx = buildSafetyContextFromElement( + input, snap.currentUrl, page.title || 'Desktop Page', 'type', + ) + proposeAction( - { type: 'type', ref: '@e3', text: 'fix: update dependencies to v3.2.1' }, + { type: 'type', ref: input.ref, text: 'Hello from Hermes Agent' }, 'agent', 'demo_task', - 'Agent wants to fill in a PR title', + `Agent wants to type into ${input.tagName} ${input.placeholder || input.id || ''}`, 'desktop-visible', - { - originUrl: page.url || 'https://github.com/gu/trendradar/pull/42', - originTitle: page.title || 'Pull Request #42', - targetDescription: "input field 'PR Title' (tag: input, type: text) inside form near heading 'Create Pull Request'", - targetRef: '@e3', - typeText: 'fix: update dependencies to v3.2.1', - riskLevel: 'medium', - } satisfies BrowserActionSafetyContext, + ctx, ) }} /> { diff --git a/apps/desktop/src/global.d.ts b/apps/desktop/src/global.d.ts index 9c6da949a6b9..89a4f1a84d1e 100644 --- a/apps/desktop/src/global.d.ts +++ b/apps/desktop/src/global.d.ts @@ -90,6 +90,8 @@ declare global { getDomSummary: () => Promise getScreenshot: () => Promise getSelectedText: () => Promise + getInteractiveSnapshot: () => Promise + executeClick: (payload: DesktopExecuteClickInput) => Promise verifyActionTarget: (payload: DesktopVerifyTargetInput) => Promise navigate: (payload: DesktopBrowserNavigatePayload) => Promise reload: () => Promise @@ -476,6 +478,47 @@ export interface DesktopBrowserNavigateResult { error?: string } +// ── Phase 2F-A2: Interactive snapshot ────────────────────────────────────── + +export interface DesktopInteractiveSnapshotElement { + ref: string + tagName: string + role: string | null + semanticRole: string + highRisk: boolean + isDestructive: boolean + isMediumRisk: boolean + textContent: string + ariaLabel: string | null + id: string | null + name: string | null + inputType: string | null + placeholder: string | null + valuePreview: string | null + href: string | null + boundingBox: { x: number; y: number; w: number; h: number } + visible: boolean + disabled: boolean + readOnly: boolean + fingerprint: { + tagName: string + textContent: string + id: string | null + name: string | null + inputType: string | null + ariaLabel: string | null + rect: { x: number; y: number; w: number; h: number } + } +} + +export interface DesktopInteractiveSnapshotResult { + ok: boolean + capturedAt: string + currentUrl: string + elements: DesktopInteractiveSnapshotElement[] + error?: string +} + // ── Phase 2F-A: Read-only target verification ───────────────────────────── export interface DesktopVerifyTargetInput { @@ -506,3 +549,41 @@ export interface DesktopVerifyTargetResult { tagName?: string detail?: string } + +// ── Phase 2F-B1: Execute click ───────────────────────────────────────────── + +export interface DesktopExecuteClickInput { + targetRef: string + originUrl: string + expectedFingerprint?: { + tagName: string + textContent: string + id: string | null + name: string | null + inputType: string | null + ariaLabel: string | null + } + reason?: string +} + +export interface DesktopExecuteClickResult { + ok: boolean + reason?: string + detail?: string + currentUrl?: string + clickedAt?: { x: number; y: number } + verification?: { + refValid: boolean + invalidationReason?: string + currentUrl?: string + currentFingerprint?: { + tagName: string + textContent: string + id: string | null + name: string | null + inputType: string | null + ariaLabel: string | null + rect?: { x: number; y: number; w: number; h: number } + } + } +}