fix(desktop): stop Computer Use from silently acting on the wrong machine over a remote gateway

agent/prompt_builder.py

@@ -531,6 +531,11 @@ def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str:
     A per-turn fact, not a platform (one session alternates between app window and HUD), so it rides the
     model-bound message, never the byte-stable system prompt. Each sentence is gated on the tool it names (an
     unknown tool name invites a hallucinated call); without read_window_below the whole note is withheld.
+
+    The one thing the note cannot know is whether computer_use reaches the window at all: on a remote
+    gateway it drives the backend host's desktop while the HUD floats over the user's. That is resolved
+    when the agent asks — read_window_below returns an `agent_host` — so the prior defers to it rather
+    than carrying a locality bit that would be stale by the time anyone read it back.
     """
     names = valid_tool_names or set()
     if "read_window_below" not in names:
@@ -547,7 +552,9 @@ def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str:
          "ago, and a single message can span both."),
         ("computer_use" in names,
          "Prefer carrying the work out in that same app — computer_use "
-         "takes its name in `app` — over pulling the task into a surface of your own."),
+         "takes its name in `app` — over pulling the task into a surface of your own, unless "
+         "read_window_below reports an `agent_host`, which means computer_use drives a different "
+         "machine than the one the window is on."),
         ("computer_use" in names and "browser_navigate" in names,
          "When the app underneath is a browser, that means driving the "
          "user's browser rather than opening yours with browser_navigate."),

apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts

@@ -2,6 +2,7 @@ import { readActivePreview } from '@/app/chat/right-rail/preview-reader'
 import { writeAgentTerminalChunk } from '@/app/right-sidebar/terminal/agent-terminal-stream'
 import { readActiveTerminal } from '@/app/right-sidebar/terminal/buffer'
 import { closeAgentTerminalByProc } from '@/app/right-sidebar/terminal/terminals'
+import { withAgentLocality } from '@/lib/agent-locality'
 import type { PreviewActAction } from '@/lib/preview-act/act-in-page'
 import type { TourAction, TourStep } from '@/lib/tour'
 import { $gateway } from '@/store/gateway'
@@ -139,11 +140,17 @@ export function handleDesktopBridgeEvent(ctx: GatewayEventContext): boolean {
     if (requestId) {
       const read = window.hermesDesktop?.readWindowBelow
 
-      const answer = (result: unknown) =>
-        $gateway.get()?.request('window.read.respond', {
+      const answer = (result: unknown) => {
+        // The window is on THIS machine; computer_use runs wherever the agent
+        // does. On a remote gateway those differ, and the answer is the only
+        // place the agent learns it — see withAgentLocality.
+        const located = withAgentLocality(result)
+
+        return $gateway.get()?.request('window.read.respond', {
           request_id: requestId,
-          text: result ? JSON.stringify(result) : ''
+          text: located ? JSON.stringify(located) : ''
         })
+      }
 
       // .catch: ipcRenderer.invoke rejects on an older shell without the
       // handler or a main-side throw — without an empty answer the tool

apps/desktop/src/app/settings/computer-use-panel.tsx

@@ -1,10 +1,13 @@
+import { useStore } from '@nanostores/react'
 import { useCallback, useEffect, useRef, useState } from 'react'
 
 import { Button } from '@/components/ui/button'
 import { getActionStatus, getComputerUseStatus, grantComputerUsePermissions } from '@/hermes'
+import { agentMachineLabel, isAgentOnAnotherMachine } from '@/lib/agent-locality'
 import { AlertTriangle, Check, ExternalLink, Loader2, RefreshCw, X } from '@/lib/icons'
 import { upsertDesktopActionTask } from '@/store/activity'
 import { notify, notifyError } from '@/store/notifications'
+import { $connection } from '@/store/session'
 import type { ComputerUseStatus } from '@/types/hermes'
 
 import { Pill } from './primitives'
@@ -32,6 +35,22 @@ function GrantIcon({ granted }: { granted: boolean | null }) {
   return <Icon className="size-3" />
 }
 
+/**
+ * The card reports the readiness of whatever host the gateway runs on. On a
+ * remote backend that is not the computer the user is looking at, and every
+ * line below ("this machine", the TCC grants, driver health) is about the other
+ * one — so say which before any of it is read as a verdict on their own screen.
+ */
+function RemoteBackendNote({ machine }: { machine: string }) {
+  return (
+    <p className="px-1 text-[0.7rem] text-muted-foreground">
+      <AlertTriangle className="mr-1 inline size-3" />
+      Computer Use runs on {machine}, the machine Hermes is connected to — not on this computer. Everything below
+      describes that desktop; the agent cannot see or click your screen from there.
+    </p>
+  )
+}
+
 function PermissionRow({ granted, label, hint }: { granted: boolean | null; label: string; hint: string }) {
   return (
     <div className="flex flex-wrap items-center justify-between gap-2 rounded-lg bg-background/55 p-2.5">
@@ -61,6 +80,9 @@ function PermissionRow({ granted, label, hint }: { granted: boolean | null; labe
  * below this card (the generic ToolsetConfigPanel).
  */
 export function ComputerUsePanel({ onConfiguredChange }: ComputerUsePanelProps) {
+  const connection = useStore($connection)
+  const remote = isAgentOnAnotherMachine(connection)
+  const machine = agentMachineLabel(connection)
   const [status, setStatus] = useState<ComputerUseStatus | null>(null)
   const [loading, setLoading] = useState(true)
   const [granting, setGranting] = useState(false)
@@ -148,31 +170,38 @@ export function ComputerUsePanel({ onConfiguredChange }: ComputerUsePanelProps)
 
   if (!status.platform_supported) {
     return (
-      <p className="px-1 text-xs text-muted-foreground">
-        Computer Use isn&apos;t supported on this platform ({status.platform}).
-      </p>
+      <div className="grid gap-2">
+        {remote && <RemoteBackendNote machine={machine} />}
+        <p className="px-1 text-xs text-muted-foreground">
+          Computer Use isn&apos;t supported on {remote ? machine : 'this machine'} ({status.platform}).
+        </p>
+      </div>
     )
   }
 
   if (!status.installed) {
     return (
-      <p className="px-1 text-xs text-muted-foreground">
-        Install the cua-driver backend below to drive this machine.
-        {status.can_grant && ' Then grant Accessibility and Screen Recording here.'}
-      </p>
+      <div className="grid gap-2">
+        {remote && <RemoteBackendNote machine={machine} />}
+        <p className="px-1 text-xs text-muted-foreground">
+          Install the cua-driver backend below to drive {remote ? machine : 'this machine'}.
+          {status.can_grant && ' Then grant Accessibility and Screen Recording here.'}
+        </p>
+      </div>
     )
   }
 
   const failingChecks = status.checks.filter(c => c.status !== 'ok')
 
   return (
     <div className="grid gap-2">
+      {remote && <RemoteBackendNote machine={machine} />}
       <div className="flex flex-wrap items-center justify-between gap-2 px-1">
         <div className="min-w-0">
           {status.can_grant ? (
             <p className="text-[0.72rem] text-muted-foreground">
               Grants attach to CuaDriver&apos;s own identity (com.trycua.driver), not Hermes — so the dialog is
-              attributed to the process that drives your Mac.
+              attributed to the process that drives {remote ? machine : 'your Mac'}.
             </p>
           ) : (
             <p className="text-[0.72rem] text-muted-foreground">{PLATFORM_NOTE[status.platform] ?? ''}</p>
@@ -225,7 +254,7 @@ export function ComputerUsePanel({ onConfiguredChange }: ComputerUsePanelProps)
       {status.ready ? (
         <div className="flex items-center gap-1.5 px-1 text-xs text-muted-foreground">
           <Check className="size-3.5" />
-          Computer Use is ready. Ask the agent to capture an app and click around.
+          Computer Use is ready{remote ? ` on ${machine}` : ''}. Ask the agent to capture an app and click around.
         </div>
       ) : (
         status.can_grant && (

apps/desktop/src/lib/agent-locality.test.ts

@@ -0,0 +1,86 @@
+import { beforeEach, describe, expect, it } from 'vitest'
+
+import { $connection } from '@/store/session'
+
+import { agentMachineLabel, isAgentOnAnotherMachine, withAgentLocality } from './agent-locality'
+
+const connection = (extra: Record<string, unknown>) => ({ baseUrl: '', token: '', wsUrl: '', ...extra }) as never
+
+const WINDOW = { window: { app: 'Figma', title: '' }, platform: 'darwin' }
+
+beforeEach(() => $connection.set(null))
+
+describe('isAgentOnAnotherMachine', () => {
+  it('is false on a local backend and with no connection yet', () => {
+    expect(isAgentOnAnotherMachine(null)).toBe(false)
+    expect(isAgentOnAnotherMachine(connection({ mode: 'local' }))).toBe(false)
+  })
+
+  it('is true for every remote shape, including a tunnelled SSH loopback URL', () => {
+    for (const remoteKind of ['ssh', 'url', 'cloud'] as const) {
+      expect(
+        isAgentOnAnotherMachine(connection({ baseUrl: 'http://127.0.0.1:41001', mode: 'remote', remoteKind }))
+      ).toBe(true)
+    }
+  })
+})
+
+describe('agentMachineLabel', () => {
+  it('is empty when the agent is already on this machine', () => {
+    expect(agentMachineLabel(connection({ mode: 'local', remoteHost: 'ignored' }))).toBe('')
+  })
+
+  it('prefers the stable SSH identity over the forwarded loopback port', () => {
+    const label = agentMachineLabel(
+      connection({
+        baseUrl: 'http://127.0.0.1:41001',
+        mode: 'remote',
+        remoteHost: 'remote-box',
+        remoteIdentity: 'operator@remote-box',
+        remoteKind: 'ssh'
+      })
+    )
+
+    expect(label).toBe('operator@remote-box')
+  })
+
+  it('names cloud and URL backends', () => {
+    expect(agentMachineLabel(connection({ mode: 'remote', remoteKind: 'cloud' }))).toBe('Hermes Cloud')
+    expect(
+      agentMachineLabel(connection({ baseUrl: 'https://nas.local:9119', mode: 'remote', remoteKind: 'url' }))
+    ).toBe('nas.local:9119')
+  })
+
+  it('always says something, even for a remote with no identifying fields', () => {
+    expect(agentMachineLabel(connection({ mode: 'remote' }))).toBe('the connected backend')
+  })
+})
+
+describe('withAgentLocality', () => {
+  it('adds nothing on a local session, so the common case costs no tokens', () => {
+    expect(withAgentLocality(WINDOW, connection({ mode: 'local' }))).toEqual(WINDOW)
+  })
+
+  it('flags the gap on a remote session without leaking where the backend is', () => {
+    const located = withAgentLocality(
+      WINDOW,
+      connection({ mode: 'remote', remoteHost: 'remote-box', remoteIdentity: 'operator@remote-box', remoteKind: 'ssh' })
+    )
+
+    expect(located).toEqual({ ...WINDOW, agent_on_this_machine: false })
+    expect(JSON.stringify(located)).not.toContain('remote-box')
+  })
+
+  it('leaves an unavailable answer alone so the tool still reports enumeration failure', () => {
+    const remote = connection({ mode: 'remote' })
+
+    expect(withAgentLocality(null, remote)).toBeNull()
+    expect(withAgentLocality('', remote)).toBe('')
+  })
+
+  it('reads the live connection when none is passed', () => {
+    $connection.set(connection({ mode: 'remote' }))
+
+    expect(withAgentLocality(WINDOW)).toEqual({ ...WINDOW, agent_on_this_machine: false })
+  })
+})

apps/desktop/src/lib/agent-locality.ts

@@ -0,0 +1,65 @@
+import { $connection } from '@/store/session'
+
+/**
+ * Whether the agent runs on a different computer than this app.
+ *
+ * Almost every Desktop-gated tool acts on the machine the user is looking at:
+ * the agent asks the renderer over the gateway bridge and the renderer answers
+ * for this computer. `computer_use` is the exception — it drives a cua-driver
+ * process on whatever host the gateway runs on. Those are the same computer on
+ * a local backend and two different computers on an SSH, URL, or cloud one.
+ *
+ * Only the client can answer this. The backend behind an SSH tunnel sees a
+ * loopback peer whether or not the person is in the room, so a server-side
+ * guess is wrong in exactly the direction that matters.
+ */
+export function isAgentOnAnotherMachine(connection = $connection.get()): boolean {
+  return connection?.mode === 'remote'
+}
+
+/** Names the host the agent runs on, for copy that must not say "this Mac".
+ *  Empty when the agent is already on this machine. */
+export function agentMachineLabel(connection = $connection.get()): string {
+  if (!connection || !isAgentOnAnotherMachine(connection)) {
+    return ''
+  }
+
+  if (connection.remoteKind === 'cloud') {
+    return 'Hermes Cloud'
+  }
+
+  const identity = connection.remoteKind === 'ssh' ? connection.remoteIdentity || connection.remoteHost : undefined
+
+  return identity || connection.remoteHost || hostOf(connection.baseUrl) || 'the connected backend'
+}
+
+/** `https://nas.local:9119` reads as a machine once the scheme is off. */
+function hostOf(baseUrl: string | undefined): string {
+  if (!baseUrl) {
+    return ''
+  }
+
+  try {
+    return new URL(baseUrl).host || baseUrl
+  } catch {
+    return baseUrl
+  }
+}
+
+/**
+ * Tell the agent whether the window we just enumerated is one it can reach.
+ *
+ * Answered when the agent asks rather than stamped on the session: a transcript
+ * outlives whoever was watching when it was written, and the same session can
+ * be reopened from another machine. Only the flag crosses — the backend names
+ * itself, so no host or connection detail leaves the client.
+ *
+ * Local sessions carry nothing, so the common case costs no tokens.
+ */
+export function withAgentLocality(result: unknown, connection = $connection.get()): unknown {
+  if (!result || typeof result !== 'object' || !isAgentOnAnotherMachine(connection)) {
+    return result
+  }
+
+  return { ...result, agent_on_this_machine: false }
+}

tests/tools/test_read_window_tool.py

@@ -48,3 +48,53 @@ def _boom():
 
     result = json.loads(rw.read_window_below_tool(callback=_boom))
     assert "renderer went away" in result["error"]
+
+
+class TestAgentHost:
+    """The window is on the user's screen; computer_use drives the gateway's
+    host. On a remote backend those are different machines, and the answer is
+    where the agent finds out — the HUD note can only tell it to look."""
+
+    WINDOW = {"window": {"app": "Figma", "title": ""}, "platform": "darwin"}
+
+    def _read(self, **extra):
+        payload = {**self.WINDOW, **extra}
+
+        return json.loads(rw.read_window_below_tool(callback=lambda: json.dumps(payload)))
+
+    def test_local_session_says_nothing(self):
+        """The desktop app omits the flag when the agent is on this machine, so
+        the common case pays no tokens for a fact that is already true."""
+        assert self._read() == self.WINDOW
+
+    def test_remote_session_is_told_it_cannot_click_the_window(self):
+        note = self._read(agent_on_this_machine=False)["agent_host"]["note"]
+
+        assert "computer_use" in note
+        assert "cannot click" in note
+
+    def test_remote_session_names_the_machine_it_is_actually_on(self, monkeypatch):
+        monkeypatch.setattr(rw.socket, "gethostname", lambda: "remote-box")
+        agent_host = self._read(agent_on_this_machine=False)["agent_host"]
+
+        assert agent_host["same_machine"] is False
+        assert agent_host["name"] == "remote-box"
+        assert "on remote-box" in agent_host["note"]
+
+    def test_an_unresolvable_hostname_still_states_the_gap(self, monkeypatch):
+        def _boom():
+            raise OSError("no hostname")
+
+        monkeypatch.setattr(rw.socket, "gethostname", _boom)
+        agent_host = self._read(agent_on_this_machine=False)["agent_host"]
+
+        assert agent_host["name"] is None
+        assert "on another machine" in agent_host["note"]
+
+    def test_the_wire_flag_is_not_passed_through_to_the_model(self):
+        """Two ways to say the same thing invites the model to trust the terser
+        one and skip the note that tells it what to do instead."""
+        assert "agent_on_this_machine" not in self._read(agent_on_this_machine=False)
+
+    def test_an_older_desktop_that_never_sends_the_flag_is_unchanged(self):
+        assert self._read(agent_on_this_machine=True) == self.WINDOW

tests/tui_gateway/test_hud_surface_note.py

@@ -72,6 +72,16 @@ def test_browser_preference_needs_a_browser_to_prefer_over(self):
         assert "computer_use" in note
         assert "browser_navigate" not in note
 
+    def test_the_drive_it_prior_defers_to_what_the_window_read_reports(self):
+        """On a remote gateway computer_use drives the backend host's desktop,
+        not the one the HUD floats over. The note cannot know which — only the
+        read_window_below answer can — so it points at that instead of
+        instructing the model to click a machine nobody is looking at."""
+        assert "agent_host" in hud_surface_note({"read_window_below", "computer_use"})
+
+    def test_nothing_to_defer_to_without_the_tool_that_can_drive(self):
+        assert "agent_host" not in hud_surface_note({"read_window_below"})
+
     def test_no_tools_at_all(self):
         assert hud_surface_note(None) == ""