feat(desktop): add read-only action target verification (Phase 2F-A)

apps/desktop/electron/main.cjs

@@ -5672,6 +5672,137 @@ ipcMain.handle('hermes:browser:is-available', async () => {
   }
 })
 
+// ── Phase 2F-A: Read-only target resolution ────────────────────────────
+//
+// This handler performs a PURELY READ-ONLY DOM query to verify that a
+// target element matching the agent's safetyContext still exists on the
+// current page.  It uses a fixed internal script — no agent-supplied JS
+// is ever evaluated.  It does NOT click, focus, dispatchEvent, or set
+// any value.
+//
+// Security:
+// - Fixed script string (not user/agent-provided).
+// - The only dynamic input is `targetRef`, validated against /^@e\d+$/.
+// - Returns element metadata only — no page mutation.
+// - No eval of arbitrary expressions.
+//
+// @see docs/architecture/desktop-browser-agent-action-safety.md §5.1
+
+ipcMain.handle('hermes:browser:verify-action-target', async (_event, payload) => {
+  try {
+    const targetRef = String(payload?.targetRef || '').trim()
+    const originUrl = String(payload?.originUrl || '').trim()
+
+    // Validate targetRef shape – must be @eN where N is one or more digits
+    if (!/^@e\d+$/.test(targetRef)) {
+      return {
+        found: false,
+        reason: 'invalid_target_ref',
+        currentUrl: '',
+        urlMatchesOrigin: false,
+      }
+    }
+
+    const view = getBrowserView()
+    const wc = view.webContents
+
+    // ── Resolve element by ref ──────────────────────────────────────────
+    // The ref @eN maps to an element with a data-agent-ref attribute or
+    // an aria attribute set by the accessibility snapshot system.
+    // We use a fixed script that queries the DOM without mutation.
+    const result = await wc.executeJavaScript(`
+      (() => {
+        const ref = ${JSON.stringify(targetRef)}
+        const origin = ${JSON.stringify(originUrl)}
+        const currentUrl = window.location.href
+
+        // Try to find the element by data-agent-ref attribute
+        let el = document.querySelector('[data-agent-ref="' + ref + '"]')
+        if (!el) {
+          // Fallback: the snapshot system may use aria attributes
+          el = document.querySelector('[aria-describedby="' + ref + '"]')
+        }
+        if (!el) {
+          // Last resort: look for elements whose computed aria label
+          // contains the ref — the ref @e5 may be embedded in an
+          // aria attribute generated by the accessibility mapper.
+          return {
+            found: false,
+            reason: 'target_not_found',
+            currentUrl: currentUrl,
+            urlMatchesOrigin: currentUrl === origin,
+          }
+        }
+
+        // ── Read element metadata (purely read-only) ──────────────────
+        const rect = el.getBoundingClientRect()
+        const style = window.getComputedStyle(el)
+        const tagName = el.tagName
+        const textContent = (el.textContent || '').trim().slice(0, 200)
+        const id = el.id || null
+        const name = el.getAttribute('name') || null
+        const inputType = (tagName === 'INPUT' || tagName === 'TEXTAREA' || tagName === 'SELECT')
+          ? (el.getAttribute('type') || 'text') : null
+        const ariaLabel = el.getAttribute('aria-label') || null
+        const value = (el.value !== undefined && el.value !== null) ? String(el.value).slice(0, 200) : null
+        const placeholder = el.getAttribute('placeholder') || null
+
+        // Visibility checks
+        const visible = (
+          rect.width > 0 &&
+          rect.height > 0 &&
+          style.visibility !== 'hidden' &&
+          style.display !== 'none' &&
+          parseFloat(style.opacity) > 0
+        )
+        const disabled = (
+          el.disabled === true ||
+          el.getAttribute('aria-disabled') === 'true' ||
+          el.getAttribute('disabled') !== null
+        )
+        const readOnly = (
+          el.readOnly === true ||
+          el.getAttribute('aria-readonly') === 'true' ||
+          el.getAttribute('readonly') !== null
+        )
+
+        return {
+          found: true,
+          reason: null,
+          currentUrl: currentUrl,
+          urlMatchesOrigin: currentUrl === origin,
+          elementFingerprint: {
+            tagName: tagName.toLowerCase(),
+            textContent: textContent,
+            id: id,
+            name: name,
+            inputType: inputType,
+            ariaLabel: ariaLabel,
+            rect: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) },
+          },
+          boundingBox: { x: Math.round(rect.x), y: Math.round(rect.y), w: Math.round(rect.width), h: Math.round(rect.height) },
+          visible: visible,
+          disabled: disabled,
+          readOnly: readOnly,
+          value: value,
+          placeholder: placeholder,
+          tagName: tagName.toLowerCase(),
+        }
+      })()
+    `)
+
+    return result
+  } catch (error) {
+    return {
+      found: false,
+      reason: 'ipc_error',
+      currentUrl: '',
+      urlMatchesOrigin: false,
+      detail: error?.message || String(error),
+    }
+  }
+})
+
 ipcMain.handle('hermes:updates:check', async () =>
   checkUpdates().catch(error => ({
     supported: true,

apps/desktop/electron/preload.cjs

@@ -125,6 +125,7 @@ contextBridge.exposeInMainWorld('hermesDesktop', {
     getDomSummary: () => ipcRenderer.invoke('hermes:browser:get-dom-summary'),
     getScreenshot: () => ipcRenderer.invoke('hermes:browser:get-screenshot'),
     getSelectedText: () => ipcRenderer.invoke('hermes:browser:get-selected-text'),
+    verifyActionTarget: payload => ipcRenderer.invoke('hermes:browser:verify-action-target', payload),
     navigate: payload => ipcRenderer.invoke('hermes:browser:navigate', payload),
     reload: () => ipcRenderer.invoke('hermes:browser:reload', { source: 'user' }),
     stop: () => ipcRenderer.invoke('hermes:browser:stop', { source: 'user' }),

apps/desktop/src/app/browser-runtime/action-gateway-ui.tsx

@@ -53,11 +53,14 @@ import {
 import {
   type DesktopBrowserBridge,
   getDesktopSnapshot,
+  VERIFICATION_FAILURE_REASONS,
+  verifyDesktopActionTarget,
 } from './desktop-visible-provider'
 import type {
   BrowserActionRequest,
   BrowserActionResult,
   BrowserActionRiskLevel,
+  PreActionVerification,
 } from './types'
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -144,9 +147,102 @@ export function BrowserActionGateway({ desktopBridge }: { desktopBridge?: Deskto
       }
     }
 
-    // ── Read-only actions — mark executed (no side effects) ──────────
-    if (actionType === 'snapshot' || actionType === 'vision' || actionType === 'get_images' || actionType === 'console') {
-      return { status: 'executed' as const }
+    // ── Click/type — pre-action verification (Phase 2F-A) ────────────
+    if (actionType === 'click' || actionType === 'type') {
+      let preActionVerification: PreActionVerification | undefined
+
+      if (request.safetyContext && desktopBridge) {
+        try {
+          preActionVerification = await verifyDesktopActionTarget(
+            desktopBridge,
+            request.safetyContext,
+          )
+        } catch {
+          preActionVerification = undefined
+        }
+      } else if (!request.safetyContext) {
+        preActionVerification = {
+          verifiedAt: new Date().toISOString(),
+          currentUrl: '',
+          refValid: false,
+          invalidationReason: VERIFICATION_FAILURE_REASONS.missing_safety_context,
+          snapshot: desktopBridge
+            ? await getDesktopSnapshot(desktopBridge).catch(() => null as unknown as PreActionVerification['snapshot'])
+            : null as unknown as PreActionVerification['snapshot'],
+        }
+      }
+
+      // Phase 2F-A does NOT execute real click/type.
+      // The verification result is attached to the log for the user to inspect.
+      return {
+        status: 'failed' as const,
+        error: `"${actionType}" execution is not yet implemented (Phase 2F-A). `
+          + (preActionVerification?.refValid
+            ? 'Pre-action verification passed — target element found and matches safety context. Ready for Phase 2F-B executor.'
+            : `Pre-action verification failed: ${preActionVerification?.invalidationReason || 'unknown'}. `),
+        preActionVerification,
+      }
+    }
+
+    // ── Snapshot — actually capture a real snapshot ──────────────────
+    if (actionType === 'snapshot') {
+      if (!desktopBridge) {
+        return {
+          status: 'failed' as const,
+          error: 'Desktop browser bridge is unavailable — cannot capture snapshot.',
+        }
+      }
+
+      try {
+        return {
+          status: 'executed' as const,
+          postActionSnapshot: await getDesktopSnapshot(desktopBridge),
+        }
+      } catch (error) {
+        return {
+          status: 'failed' as const,
+          error: `Snapshot capture failed: ${error instanceof Error ? error.message : String(error)}`,
+        }
+      }
+    }
+
+    // ── Vision — capture screenshot (read-only, no AI analysis) ──────
+    if (actionType === 'vision') {
+      if (!desktopBridge) {
+        return {
+          status: 'failed' as const,
+          error: 'Desktop browser bridge is unavailable — cannot capture screenshot.',
+        }
+      }
+
+      try {
+        const screenshot = await desktopBridge.getScreenshot()
+
+        return {
+          status: 'executed' as const,
+          screenshotRef: screenshot?.dataURL || undefined,
+        }
+      } catch (error) {
+        return {
+          status: 'failed' as const,
+          error: `Screenshot capture failed: ${error instanceof Error ? error.message : String(error)}`,
+        }
+      }
+    }
+
+    // ── get_images / console — acknowledged but not implemented ──────
+    if (actionType === 'get_images') {
+      return {
+        status: 'failed' as const,
+        error: 'get_images is not available on the Desktop browser. Use snapshot to read page content.',
+      }
+    }
+
+    if (actionType === 'console') {
+      return {
+        status: 'failed' as const,
+        error: 'Console reading is not available on the Desktop browser.',
+      }
     }
 
     // ── Everything else: not executable ─────────────────────────────
@@ -486,6 +582,12 @@ function ActionLogEntry({ entry }: { entry: BrowserActionResultLike }) {
         ? 'bg-amber-500/10 text-amber-600'
         : 'bg-blue-500/10 text-blue-600'
 
+  const verificationLabel = entry.preActionVerification
+    ? (entry.preActionVerification.refValid
+      ? 'Verified'
+      : `Verify Failed: ${entry.preActionVerification.invalidationReason || 'unknown'}`)
+    : undefined
+
   return (
     <div className="rounded px-1.5 py-0.5 text-[0.625rem]">
       <div className="flex items-center gap-1.5">
@@ -498,7 +600,17 @@ function ActionLogEntry({ entry }: { entry: BrowserActionResultLike }) {
         <span className="min-w-0 truncate text-foreground/80">
           {entry.error || entry.executedBy.provider}
         </span>
-        <span className="shrink-0 tabular-nums text-muted-foreground/40">
+        {verificationLabel && (
+          <span className={cn(
+            'shrink-0 rounded-full px-1 py-0 text-[0.5rem] font-medium',
+            entry.preActionVerification?.refValid
+              ? 'bg-emerald-500/10 text-emerald-600'
+              : 'bg-red-500/10 text-red-600',
+          )}>
+            {verificationLabel}
+          </span>
+        )}
+        <span className="shrink-0 tabular-nums text-muted-foreground/40 ml-auto">
           {new Date(entry.executedAt).toLocaleTimeString()}
         </span>
       </div>
@@ -650,4 +762,15 @@ export type BrowserActionResultLike = {
   error?: string
   /** Action type for display (filled at log time). */
   actionType?: string
+  /** Phase 2F-A: Pre-action verification result. */
+  preActionVerification?: {
+    verifiedAt: string
+    refValid: boolean
+    invalidationReason?: string
+    currentUrl?: string
+    currentFingerprint?: {
+      tagName: string
+      textContent: string
+    }
+  }
 }