From 4996448a39e793dd755ac4b64ae50855b806c181 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:20:27 -0700 Subject: [PATCH 01/55] =?UTF-8?q?feat:=20Bot=20Screen=20=E2=80=94=20per-bo?= =?UTF-8?q?t=20Xfce=20desktop=20streamed=20into=20Hermes=20Desktop=20with?= =?UTF-8?q?=20human=20take-over?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bot running on a headless Linux gateway now gets its own desktop (TigerVNC Xvnc + a minimal Xfce session, one per profile) that Hermes Desktop streams live. The user can watch the bot work, take over to type a login / 2FA code / CAPTCHA, and hand control back; the bot refuses every computer_use action (screenshots included) while a human holds the screen, then resumes with the session cookies the human just created. Why this shape: - The screen lives on the machine Hermes runs on, not in a vendor cloud browser, so it works for any app the bot drives and keeps the session on the user's host. - Xfce components are launched individually (xfwm4, xfce4-panel, xfdesktop, xfsettingsd) under a private dbus session rather than xfce4-session/the metapackage: no screensaver, power manager or polkit agent to lock or prompt a headless desktop. - Transport is raw RFB over a WebSocket beside /api/ws, authenticated with a one-shot ticket minted through the already-authenticated RPC channel; noVNC runs in the Electron renderer. The bridge parses the RFB client stream and drops keyboard/pointer/clipboard (incl. QEMU Extended KeyEvent, which noVNC switches to once Xvnc advertises it) from any viewer that does not hold the lease, so viewOnly is enforced server-side, not by the client. - One lease per profile (agent | human viewer) is the single truth for the RFB bridge, the computer_use tool and the Desktop UI; taking control evicts other viewers' input with close code 4000 control-taken. - Auto-start happens only at the computer_use tool boundary (headless host, packages present, bot_desktop.auto_start=true); env builders stay pure so status probes and tests never spawn X servers. tests/tools/conftest.py pins the binaries to "missing" for the same reason the browser-use fixture does. Surfaces: Desktop (Bots → right-click → Open Screen; Take over / Hand back), CLI (`hermes computer-use screen status|start|stop|install-deps`), tool (`computer_use` actions request_handoff / wait_for_human), gateway RPCs (display.status/start/stop/observe/lease.acquire/lease.release + display.lease event), docs page user-guide/features/bot-screen. --- apps/desktop/package.json | 1 + apps/desktop/src/lib/sibling-ws-url.test.ts | 49 +++ apps/desktop/src/lib/sibling-ws-url.ts | 86 +++++ .../src/plugins/hermes-bots/bot-row.tsx | 2 + apps/desktop/src/plugins/hermes-bots/i18n.ts | 117 +++++++ .../plugins/hermes-bots/screen-connection.ts | 81 +++++ .../src/plugins/hermes-bots/screen-open.tsx | 45 +++ .../src/plugins/hermes-bots/screen-pane.tsx | 306 ++++++++++++++++++ .../src/plugins/hermes-bots/screen-state.ts | 40 +++ apps/desktop/src/sdk/index.ts | 3 + hermes_cli/config_defaults.py | 7 + hermes_cli/dashboard_auth/ws_tickets.py | 8 +- hermes_cli/subcommands/computer_use.py | 8 +- hermes_cli/subcommands/computer_use_screen.py | 108 +++++++ hermes_cli/web_routers/display.py | 144 +++++++++ hermes_cli/web_server.py | 2 + package-lock.json | 9 +- tests/hermes_cli/test_display_ws_ticket.py | 27 ++ tests/tools/conftest.py | 18 ++ tests/tools/test_bot_desktop_lease.py | 67 ++++ tools/bot_desktop/__init__.py | 5 + tools/bot_desktop/launcher.sh | 153 +++++++++ tools/bot_desktop/lease.py | 154 +++++++++ tools/bot_desktop/rfb_filter.py | 94 ++++++ tools/bot_desktop/runtime.py | 295 +++++++++++++++++ tools/browser_tool.py | 4 +- tools/browser_tool_real_profile.py | 5 +- tools/computer_use/cua_backend.py | 4 + tools/computer_use/handoff.py | 38 +++ tools/computer_use/schema.py | 9 +- tools/computer_use/tool.py | 11 + tui_gateway/methods_display.py | 121 +++++++ tui_gateway/server.py | 5 +- .../docs/user-guide/features/bot-screen.md | 121 +++++++ .../docs/user-guide/features/computer-use.md | 9 +- website/sidebars.ts | 1 + 36 files changed, 2141 insertions(+), 16 deletions(-) create mode 100644 apps/desktop/src/lib/sibling-ws-url.test.ts create mode 100644 apps/desktop/src/lib/sibling-ws-url.ts create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-connection.ts create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-open.tsx create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-pane.tsx create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-state.ts create mode 100644 hermes_cli/subcommands/computer_use_screen.py create mode 100644 hermes_cli/web_routers/display.py create mode 100644 tests/hermes_cli/test_display_ws_ticket.py create mode 100644 tests/tools/test_bot_desktop_lease.py create mode 100644 tools/bot_desktop/__init__.py create mode 100755 tools/bot_desktop/launcher.sh create mode 100644 tools/bot_desktop/lease.py create mode 100644 tools/bot_desktop/rfb_filter.py create mode 100644 tools/bot_desktop/runtime.py create mode 100644 tools/computer_use/handoff.py create mode 100644 tui_gateway/methods_display.py create mode 100644 website/docs/user-guide/features/bot-screen.md diff --git a/apps/desktop/package.json b/apps/desktop/package.json index b21be619de87..b9956ef30f1a 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -96,6 +96,7 @@ "@icons-pack/react-simple-icons": "13.11.1", "@lezer/highlight": "1.2.3", "@nanostores/react": "1.1.0", + "@novnc/novnc": "1.7.0", "@nous-research/ui": "0.18.2", "@streamdown/code": "1.1.1", "@streamdown/math": "1.0.2", diff --git a/apps/desktop/src/lib/sibling-ws-url.test.ts b/apps/desktop/src/lib/sibling-ws-url.test.ts new file mode 100644 index 000000000000..06ac27e43edf --- /dev/null +++ b/apps/desktop/src/lib/sibling-ws-url.test.ts @@ -0,0 +1,49 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { resolveSiblingWsUrl } from './sibling-ws-url' + +// A sibling stream (voice PCM, Bot Screen RFB) must dial the SAME (connection, +// profile) backend chat uses. The bare v1 getConnection pair answers for the +// local primary — the wrong machine when a registry remote rides over a local +// install — so registry routes must go through the *For bridges. +describe('resolveSiblingWsUrl', () => { + const remoteWsUrl = 'wss://gateway.example/api/ws?ticket=fresh' + const localWsUrl = 'ws://127.0.0.1:5151/api/ws?token=local' + + let getConnection: ReturnType + let getConnectionFor: ReturnType + let getGatewayWsUrl: ReturnType + let getGatewayWsUrlFor: ReturnType + + beforeEach(() => { + getConnection = vi.fn(async () => ({ authMode: 'token', baseUrl: 'http://127.0.0.1:5151', wsUrl: localWsUrl })) + getConnectionFor = vi.fn(async () => ({ authMode: 'token', baseUrl: 'https://gateway.example', wsUrl: remoteWsUrl })) + getGatewayWsUrl = vi.fn(async () => ({ ok: true, wsUrl: localWsUrl })) + getGatewayWsUrlFor = vi.fn(async () => ({ ok: true, wsUrl: remoteWsUrl })) + Object.defineProperty(window, 'hermesDesktop', { + configurable: true, + value: { getConnection, getConnectionFor, getGatewayWsUrl, getGatewayWsUrlFor } + }) + }) + + afterEach(() => { + Reflect.deleteProperty(window, 'hermesDesktop') + }) + + it('routes a registry-scoped profile through the *For bridges and swaps only the path', async () => { + const url = await resolveSiblingWsUrl({ connectionId: 'gw-tailscale', profile: 'research' }, '/api/display/ws') + + expect(url).toBe('wss://gateway.example/api/display/ws?ticket=fresh') + expect(getConnectionFor).toHaveBeenCalledWith({ connectionId: 'gw-tailscale', profile: 'research' }) + expect(getConnection).not.toHaveBeenCalled() + expect(getGatewayWsUrl).not.toHaveBeenCalled() + }) + + it('strips the spent gateway credential when the sibling route authenticates itself', async () => { + const url = new URL(await resolveSiblingWsUrl({ profile: null }, 'api/display/ws', { stripGatewayCredential: true })) + + expect(url.origin + url.pathname).toBe('ws://127.0.0.1:5151/api/display/ws') + expect(url.searchParams.has('token')).toBe(false) + expect(url.searchParams.has('ticket')).toBe(false) + }) +}) diff --git a/apps/desktop/src/lib/sibling-ws-url.ts b/apps/desktop/src/lib/sibling-ws-url.ts new file mode 100644 index 000000000000..7ba0be8045be --- /dev/null +++ b/apps/desktop/src/lib/sibling-ws-url.ts @@ -0,0 +1,86 @@ +/** + * Sibling WebSocket URLs beside `/api/ws` for a (connection, profile) route. + * + * Some gateway streams are not JSON-RPC and cannot share the gateway socket: + * voice PCM (`/api/audio/speak-stream`) and the Bot Screen's raw RFB + * (`/api/display/ws`). They ride the SAME authenticated origin the route's + * `/api/ws` uses — a fresh credential for OAuth remotes, the registry-scoped + * `*For` bridges for a remote riding over a local install (the bare + * getConnection/getGatewayWsUrl pair answers for the v1 primary backend, which + * would be the wrong machine). One resolver so every sibling stream routes the + * way chat does. + */ + +import { resolveGatewayWsUrl } from '@hermes/shared' + +const RESOLVE_TIMEOUT_MS = 15_000 + +function withTimeout(promise: Promise, ms: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = window.setTimeout(() => reject(new Error(label)), ms) + promise.then( + value => { + window.clearTimeout(timer) + resolve(value) + }, + error => { + window.clearTimeout(timer) + reject(error instanceof Error ? error : new Error(String(error))) + } + ) + }) +} + +export interface SiblingWsRoute { + connectionId?: null | string + profile?: null | string +} + +/** + * Resolve `ws(s):///` for `route`. `stripGatewayCredential` + * drops the `?ticket=`/`?token=` the gateway URL carried when the sibling route + * authenticates on its own credential (a one-shot ticket must not be spent + * twice; the display bridge mints its own). + */ +export async function resolveSiblingWsUrl( + route: SiblingWsRoute, + path: string, + options: { stripGatewayCredential?: boolean } = {} +): Promise { + const desktop = window.hermesDesktop + + if (!desktop?.getConnection) { + throw new Error('Hermes Desktop connection bridge unavailable') + } + + const connectionId = route.connectionId?.trim() || null + const profile = route.profile?.trim() || null + + const conn = + connectionId && desktop.getConnectionFor + ? await withTimeout(desktop.getConnectionFor({ connectionId, profile }), RESOLVE_TIMEOUT_MS, `Timed out connecting to profile "${profile}"`) + : await withTimeout(desktop.getConnection(profile), RESOLVE_TIMEOUT_MS, `Timed out connecting to profile "${profile}"`) + + const wsDeps = + connectionId && desktop.getGatewayWsUrlFor + ? { getGatewayWsUrl: () => desktop.getGatewayWsUrlFor!({ connectionId, profile }) } + : connectionId + ? {} + : desktop + + const wsUrl = await withTimeout(resolveGatewayWsUrl(wsDeps, conn), RESOLVE_TIMEOUT_MS, 'Timed out minting the gateway WebSocket URL') + const url = new URL(wsUrl) + + if (!url.pathname.endsWith('/api/ws')) { + throw new Error(`Unexpected gateway WebSocket path: ${url.pathname}`) + } + + url.pathname = url.pathname.replace(/\/api\/ws$/, path.startsWith('/') ? path : `/${path}`) + + if (options.stripGatewayCredential) { + url.searchParams.delete('ticket') + url.searchParams.delete('token') + } + + return url.toString() +} diff --git a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx index f21a1210264d..ab4ed243729c 100644 --- a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx +++ b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx @@ -72,6 +72,7 @@ import { useTurnBusy, workerActiveAt } from './row-helpers' +import { openBotScreen } from './screen-open' import type { GroupMember, RosterRow, SidebarRowLabels } from './types' import { $botSections, $draggingBot, BOT_DRAG_MIME, botSectionId, moveBotsToSection } from './user-sections' @@ -310,6 +311,7 @@ export function BotRow({ bot, onDelete, onEdit, onGroup, onNewSection, showHandl {row} void openRosterBot(bot)}>{b.bot.openBotChat} + openBotScreen(bot, meta)}>{b.screen.menu} { diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index d6b96cf348bc..12e63a6b71fc 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -223,6 +223,31 @@ type BotsMessages = { noMcpServers: string } + /** Bot Screen: the bot's headless desktop on the gateway host, live in a pane. */ + screen: { + title: string + menu: string + unsupportedTitle: string + unsupportedBody: string + notInstalledTitle: string + notInstalledBody: string + installHint: string + recheck: string + stoppedTitle: string + stoppedBody: string + start: string + attaching: string + streamLost: string + reconnect: string + takeOver: string + handBack: string + youControl: string + otherControls: string + agentControls: string + controlTaken: string + handoffRequested: string + } + /** Bot-scoped scheduled jobs. Generic scheduling chrome (weekday names, * Daily/Hourly, the job verbs) resolves against core's `cron` section. */ cron: { @@ -446,6 +471,29 @@ const en: BotsMessages = { searchHub: 'Search the hub (community + well-known sources)…', noMcpServers: 'No MCP servers configured or in the catalog.' }, + screen: { + title: 'Screen', + menu: 'Open Screen', + unsupportedTitle: 'No bot screen on this host', + unsupportedBody: 'Bot screens run on Linux gateway hosts. This bot uses the host\u2019s own display.', + notInstalledTitle: 'Screen packages missing', + notInstalledBody: 'The gateway host needs TigerVNC and the Xfce core to give this bot a screen. Run on the host:', + installHint: 'or: hermes computer-use screen install', + recheck: 'Check again', + stoppedTitle: 'Screen is off', + stoppedBody: 'Start this bot\u2019s desktop to watch what it does and take over when it needs you.', + start: 'Start screen', + attaching: 'Connecting to the screen\u2026', + streamLost: 'Screen stream ended', + reconnect: 'Reconnect', + takeOver: 'Take over', + handBack: 'Hand back', + youControl: 'You are in control', + otherControls: 'Another viewer is in control', + agentControls: 'Bot is in control', + controlTaken: 'Another viewer took control. Watching only.', + handoffRequested: 'Bot needs you' + }, cron: { filterHint: 'Scheduled jobs exist in this profile but none are tagged for this bot. Name a job "[bot:] …" to show it here, or see them in Cron below.', @@ -665,6 +713,29 @@ const ja: BotsMessages = { searchHub: 'ハブを検索（コミュニティと既知のソース）…', noMcpServers: '設定済みまたはカタログ内の MCP サーバーはありません。' }, + screen: { + title: '画面', + menu: '画面を開く', + unsupportedTitle: 'このホストにはボット画面がありません', + unsupportedBody: 'ボット画面は Linux のゲートウェイホストで動作します。このボットはホスト自身のディスプレイを使います。', + notInstalledTitle: '画面パッケージが不足しています', + notInstalledBody: 'このボットに画面を与えるには、ゲートウェイホストに TigerVNC と Xfce コアが必要です。ホストで実行:', + installHint: 'または: hermes computer-use screen install', + recheck: '再確認', + stoppedTitle: '画面はオフです', + stoppedBody: 'このボットのデスクトップを起動すると、動作を見守り、必要なときに操作を引き継げます。', + start: '画面を起動', + attaching: '画面に接続中…', + streamLost: '画面ストリームが終了しました', + reconnect: '再接続', + takeOver: '引き継ぐ', + handBack: '戻す', + youControl: 'あなたが操作中', + otherControls: '別のビューアが操作中', + agentControls: 'ボットが操作中', + controlTaken: '別のビューアが操作を引き継ぎました。閲覧のみ。', + handoffRequested: 'ボットが助けを求めています' + }, cron: { filterHint: 'このプロファイルには定期実行ジョブがありますが、このボット向けのタグが付いたものはありません。ジョブ名を「[bot:<名前>] …」にするとここに表示されます。下のCronでも確認できます。', @@ -879,6 +950,29 @@ const zh: BotsMessages = { searchHub: '搜索技能中心（社区和常见来源）…', noMcpServers: '未配置 MCP 服务器，目录中也没有。' }, + screen: { + title: '屏幕', + menu: '打开屏幕', + unsupportedTitle: '此主机没有机器人屏幕', + unsupportedBody: '机器人屏幕在 Linux 网关主机上运行。此机器人使用主机自身的显示器。', + notInstalledTitle: '缺少屏幕软件包', + notInstalledBody: '网关主机需要 TigerVNC 和 Xfce 核心组件才能为此机器人提供屏幕。在主机上运行:', + installHint: '或: hermes computer-use screen install', + recheck: '重新检查', + stoppedTitle: '屏幕已关闭', + stoppedBody: '启动此机器人的桌面，观看它的操作，并在需要时接管。', + start: '启动屏幕', + attaching: '正在连接屏幕…', + streamLost: '屏幕流已结束', + reconnect: '重新连接', + takeOver: '接管', + handBack: '交还', + youControl: '你正在控制', + otherControls: '另一位查看者正在控制', + agentControls: '机器人正在控制', + controlTaken: '另一位查看者已接管控制。仅可观看。', + handoffRequested: '机器人需要你' + }, cron: { filterHint: '此配置档案中有定时任务，但没有一个标记给这个机器人。将任务命名为“[bot:<名称>] …”即可显示在这里，也可以在下方的 Cron 中查看。', @@ -1093,6 +1187,29 @@ const zhHant: BotsMessages = { searchHub: '搜尋技能中心（社群和常見來源）…', noMcpServers: '未設定 MCP 伺服器，目錄中也沒有。' }, + screen: { + title: '螢幕', + menu: '開啟螢幕', + unsupportedTitle: '此主機沒有機器人螢幕', + unsupportedBody: '機器人螢幕在 Linux 閘道主機上執行。此機器人使用主機自身的顯示器。', + notInstalledTitle: '缺少螢幕套件', + notInstalledBody: '閘道主機需要 TigerVNC 與 Xfce 核心元件才能為此機器人提供螢幕。在主機上執行:', + installHint: '或: hermes computer-use screen install', + recheck: '重新檢查', + stoppedTitle: '螢幕已關閉', + stoppedBody: '啟動此機器人的桌面，觀看它的操作，並在需要時接手。', + start: '啟動螢幕', + attaching: '正在連線至螢幕…', + streamLost: '螢幕串流已結束', + reconnect: '重新連線', + takeOver: '接手', + handBack: '交還', + youControl: '你正在控制', + otherControls: '另一位檢視者正在控制', + agentControls: '機器人正在控制', + controlTaken: '另一位檢視者已接手控制。僅可觀看。', + handoffRequested: '機器人需要你' + }, cron: { filterHint: '此設定檔中有排程工作，但沒有任何一個標記給這個機器人。將工作命名為「[bot:<名稱>] …」即可顯示在這裡，也可以在下方的 Cron 中查看。', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts new file mode 100644 index 000000000000..247a79ec0e04 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -0,0 +1,81 @@ +/** + * Bot Screen — connection plumbing for a bot's Bot Desktop (the headless Xfce + * screen its computer_use drives on the gateway host). + * + * Two legs share one authenticated origin: JSON-RPC (`display.*`) rides the + * bot's pooled gateway socket through `host.requestProfile`; the RFB stream + * rides a SIBLING WebSocket to `/api/display/ws`, minted per attach by + * `display.observe` (single-use, 30 s). noVNC's Websock takes ownership of the + * socket it is handed, so it can never share the JSON-RPC one — same reason + * voice playback opens `/api/audio/speak-stream` beside `/api/ws`. + */ + +import { host, resolveSiblingWsUrl } from '@hermes/plugin-sdk' +import type { PluginProfileRoute } from '@hermes/plugin-sdk' + +import { botConnectionRoute } from './routing' +import type { RosterRow } from './types' + +export interface DisplayLease { + holder: 'agent' | 'human' + viewer_id: null | string + since: number + reason: string + pending_handoff: null | string +} + +export interface DisplayStatus { + profile: string + profile_key: string + supported: boolean + installed: boolean + missing: string[] + running: boolean + pid: null | number + display: null | string + socket: null | string + geometry: string + install_command: null | string + lease: DisplayLease +} + +export interface DisplayObserveResult extends DisplayStatus { + ticket: string + path: string + viewer_id: string +} + +/** Stable per-window viewer identity: the lease names who holds control, and a + * reload must NOT silently inherit a stale holder's authority. */ +export const VIEWER_ID = `desktop-${Math.random().toString(36).slice(2, 10)}` + +/** Bare-profile fallback so a v1 local bot (no registry route) still resolves. */ +export function botScreenRoute(bot: RosterRow): PluginProfileRoute | string { + return botConnectionRoute(bot) ?? bot.name +} + +export function displayRequest(bot: RosterRow, method: string, params: Record = {}): Promise { + return host.requestProfile(botScreenRoute(bot), method, params) +} + +/** + * Resolve the RFB WebSocket URL for `bot`: the bot's gateway `/api/ws` origin + * (fresh credential for OAuth remotes) with the path swapped for the display + * bridge and the single-use display ticket attached. + */ +export async function resolveScreenWsUrl(bot: RosterRow, ticket: string): Promise { + const route = botConnectionRoute(bot) + + // The /api/ws credential authenticated the RPC that minted the display ticket; + // the bridge authenticates on the ticket alone, so the gateway credential is + // dropped rather than spending a second one-shot ticket. + const url = new URL( + await resolveSiblingWsUrl({ connectionId: route?.connectionId ?? null, profile: route?.profile ?? bot.name }, '/api/display/ws', { + stripGatewayCredential: true + }) + ) + + url.searchParams.set('display_ticket', ticket) + + return url.toString() +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-open.tsx b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx new file mode 100644 index 000000000000..ae0ff946c64f --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx @@ -0,0 +1,45 @@ +/** + * Open a bot's Screen as a main-window workspace tab (host.openWorkspace), + * one tab per bot; a second open refocuses the existing tab. + */ + +import { host } from '@hermes/plugin-sdk' + +import { botSelectionKey } from './data' +import { displayName } from './labels' +import { BotScreenPane } from './screen-pane' +import { ID } from './shared' +import type { BotMeta, RosterRow } from './types' + +const openTabs = new Map void>() + +export function screenPaneId(bot: RosterRow): string { + return `plugin-workspace:${ID}:screen:${botSelectionKey(bot)}` +} + +export function openBotScreen(bot: RosterRow, meta?: BotMeta | null): void { + if (typeof host.openWorkspace !== 'function') { + host.notify({ kind: 'info', message: 'Update Hermes Desktop to open bot screens.' }) + + return + } + + const key = botSelectionKey(bot) + + if (openTabs.has(key)) { + host.revealPane(screenPaneId(bot)) + + return + } + + const close = host.openWorkspace(`${ID}:screen:${key}`, { + title: `${displayName(bot, meta ?? null)} · Screen`, + minWidth: '28rem', + render: () => , + onClose: () => { + openTabs.delete(key) + } + }) + + openTabs.set(key, close) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx new file mode 100644 index 000000000000..27d4190f9a99 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -0,0 +1,306 @@ +/** + * Bot Screen pane — live view of a bot's headless desktop with Take over / Hand back. + * + * State authority: the BACKEND owns runtime + lease (`display.status`, pushed + * as `display.lease` events); this pane paints a cache of it. The RFB stream + * is a sibling WebSocket handed to noVNC's RFB; `viewOnly` here is UX only — + * the gateway drops input from anyone but the lease holder. + * + * Every attach spends a single-use ticket, so a lease flip that the bridge + * answers with close 4000 (`control-taken`) simply re-attaches in watch mode. + */ + +import { Button, Codicon, EmptyState, GlyphSpinner, host, useValue } from '@hermes/plugin-sdk' +import type { RpcEvent } from '@hermes/plugin-sdk' +import { useCallback, useEffect, useRef, useState } from 'react' + +import { useBots } from './i18n' +import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, resolveScreenWsUrl, VIEWER_ID } from './screen-connection' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' +import type { RosterRow } from './types' + +type RfbLike = { + viewOnly: boolean + scaleViewport: boolean + resizeSession: boolean + focusOnClick: boolean + background: string + qualityLevel: number + addEventListener: (type: string, handler: (event: { detail?: { clean?: boolean; reason?: string } }) => void) => void + disconnect: () => void + focus: () => void +} + +type ConnState = 'idle' | 'attaching' | 'live' | 'control-taken' | 'error' + +async function loadRfb(): Promise) => RfbLike> { + const mod = (await import('@novnc/novnc')) as unknown as { default: new (...args: never[]) => RfbLike } + + return mod.default as unknown as new (target: HTMLElement, socket: WebSocket, options?: Record) => RfbLike +} + +export function BotScreenPane({ bot }: { bot: RosterRow }) { + const t = useBots() + const screen = useValue($screenState) + const state = screenStateFor(screen, bot) + const status = state?.status ?? null + const lease = state?.lease ?? status?.lease ?? null + const iHold = lease?.holder === 'human' && lease.viewer_id === VIEWER_ID + + const canvasHost = useRef(null) + const rfb = useRef(null) + const socket = useRef(null) + const [conn, setConn] = useState('idle') + const [error, setError] = useState(null) + const [busy, setBusy] = useState(false) + const attachGeneration = useRef(0) + + const refresh = useCallback(async () => { + try { + const next = await displayRequest(bot, 'display.status') + setScreenStatus(bot, next) + setError(null) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } + }, [bot]) + + useEffect(() => { + void refresh() + + return host.onEvent('display.lease', (event: RpcEvent) => { + const payload = event.payload as { profile_key?: string; lease?: DisplayLease } | undefined + + if (payload?.lease && payload.profile_key && payload.profile_key === status?.profile_key) { + setScreenLease(bot, payload.lease) + } + }) + }, [bot, refresh, status?.profile_key]) + + const detach = useCallback(() => { + attachGeneration.current += 1 + rfb.current?.disconnect() + rfb.current = null + socket.current?.close() + socket.current = null + }, []) + + const attach = useCallback(async () => { + if (!canvasHost.current) { + return + } + + detach() + const generation = attachGeneration.current + setConn('attaching') + setError(null) + + try { + // Load the client BEFORE dialing: noVNC's Websock installs its own `onopen`, so a socket that + // opened while the dynamic import was still in flight never hands it the open event. + const Rfb = await loadRfb() + const observe = await displayRequest(bot, 'display.observe', { viewer_id: VIEWER_ID }) + setScreenStatus(bot, observe) + const url = await resolveScreenWsUrl(bot, observe.ticket) + + if (generation !== attachGeneration.current || !canvasHost.current) { + return + } + + const ws = new WebSocket(url) + ws.binaryType = 'arraybuffer' + socket.current = ws + const client = new Rfb(canvasHost.current, ws, { shared: true }) + client.scaleViewport = true + client.resizeSession = false + client.focusOnClick = true + client.background = 'transparent' + client.qualityLevel = 7 + client.viewOnly = !(observe.lease.holder === 'human' && observe.lease.viewer_id === VIEWER_ID) + client.addEventListener('connect', () => { + if (generation === attachGeneration.current) { + setConn('live') + } + }) + client.addEventListener('disconnect', event => { + if (generation !== attachGeneration.current) { + return + } + + const reason = event.detail?.reason ?? '' + + if (reason.includes('control-taken')) { + setConn('control-taken') + } else if (event.detail?.clean) { + setConn('idle') + } else { + setConn('error') + setError(reason || t.screen.streamLost) + } + + void refresh() + }) + rfb.current = client + } catch (err) { + if (generation === attachGeneration.current) { + setConn('error') + setError(err instanceof Error ? err.message : String(err)) + } + } + }, [bot, detach, refresh, t.screen.streamLost]) + + // Visibility is not lifecycle: the stream stays attached while the pane is + // hidden; only unmount tears it down (and hands control back server-side). + useEffect(() => () => detach(), [detach]) + + useEffect(() => { + if (status?.running && conn === 'idle') { + void attach() + } + }, [attach, conn, status?.running]) + + useEffect(() => { + if (rfb.current) { + rfb.current.viewOnly = !iHold + + if (iHold) { + rfb.current.focus() + } + } + }, [iHold]) + + const start = useCallback(async () => { + setBusy(true) + + try { + const next = await displayRequest(bot, 'display.start') + setScreenStatus(bot, next) + setConn('idle') + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [bot]) + + const takeOver = useCallback(async () => { + setBusy(true) + + try { + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.acquire', { viewer_id: VIEWER_ID }) + setScreenLease(bot, result.lease) + + if (conn !== 'live') { + void attach() + } + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [attach, bot, conn]) + + const handBack = useCallback(async () => { + setBusy(true) + + try { + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', { viewer_id: VIEWER_ID }) + setScreenLease(bot, result.lease) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [bot]) + + if (status && !status.supported) { + return + } + + if (status && !status.installed) { + return ( +
+
+
{t.screen.notInstalledTitle}
+
{t.screen.notInstalledBody}
+ {status.install_command ? ( + {status.install_command} + ) : null} +
{t.screen.installHint}
+ +
+
+ ) + } + + if (status && !status.running) { + return ( +
+
+
{t.screen.stoppedTitle}
+
{t.screen.stoppedBody}
+ + {error ?
{error}
: null} +
+
+ ) + } + + const humanOther = lease?.holder === 'human' && !iHold + + return ( +
+
+ + {t.screen.title} + {status?.display ? {status.display} · {status.geometry} : null} + + {lease?.pending_handoff ? ( + + {t.screen.handoffRequested} + + ) : null} + {iHold ? ( + {t.screen.youControl} + ) : humanOther ? ( + {t.screen.otherControls} + ) : ( + {t.screen.agentControls} + )} + {iHold ? ( + + ) : ( + + )} + +
+
+ {/* data-terminal: the same keyboard-ownership marker the terminal pane uses, so the app's + type-to-focus / bare-key shortcuts never steal keystrokes meant for the remote screen. */} +
+ {conn === 'attaching' ? ( +
+ {t.screen.attaching} +
+ ) : null} + {conn === 'control-taken' ? ( +
{t.screen.controlTaken}
+ ) : null} + {conn === 'error' && error ? ( +
{error}
+ ) : null} +
+
+ ) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.ts new file mode 100644 index 000000000000..e93ddea0ef75 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.ts @@ -0,0 +1,40 @@ +/** + * Per-bot Bot Screen cache: backend truth (`display.status`) plus the last + * `display.lease` event, keyed by the bot's roster identity. Renderer-owned + * cache of backend state — never the authority. + */ + +import { atom } from 'nanostores' + +import { botSelectionKey } from './data' +import type { DisplayLease, DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +export interface BotScreenState { + status: DisplayStatus | null + lease: DisplayLease | null +} + +export const $screenState = atom>({}) + +export function screenStateFor(all: Record, bot: RosterRow): BotScreenState | null { + return all[botSelectionKey(bot)] ?? null +} + +export function setScreenStatus(bot: RosterRow, status: DisplayStatus): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + $screenState.set({ ...current, [key]: { status, lease: status.lease ?? current[key]?.lease ?? null } }) +} + +export function setScreenLease(bot: RosterRow, lease: DisplayLease): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + + if (prev?.lease && prev.lease.holder === lease.holder && prev.lease.viewer_id === lease.viewer_id && prev.lease.pending_handoff === lease.pending_handoff) { + return + } + + $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease } }) +} diff --git a/apps/desktop/src/sdk/index.ts b/apps/desktop/src/sdk/index.ts index 4ce472aa508d..b3b2f85abf01 100644 --- a/apps/desktop/src/sdk/index.ts +++ b/apps/desktop/src/sdk/index.ts @@ -1721,6 +1721,9 @@ export const TITLEBAR_AREAS = { center: 'titleBar.center', left: 'titleBar.left' * setup.runtime_check, reconciled) — pass `host.request`. Don't hand-roll * readiness from raw RPC shapes. */ export { evaluateRuntimeReadiness, type RuntimeReadinessResult } from '@/lib/runtime-readiness' +/** A sibling WebSocket beside the route's `/api/ws` (voice PCM, Bot Screen RFB): + * same origin, same auth resolution as chat. */ +export { resolveSiblingWsUrl, type SiblingWsRoute } from '@/lib/sibling-ws-url' /** Canonical time formatting — every surface pulls from here so timestamps read * the same app-wide. For a row's AGE, bucket with `coarseElapsed` and render * the compact suffixes (`t.sidebar.row.ageMin` → "52m"), which is what the diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 7f850a542364..746a88ffb393 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2242,6 +2242,13 @@ def _aux(timeout, *, reasoning_effort=True, **extra): "paste_collapse_threshold_fallback": 5, "paste_collapse_char_threshold": 2000, + # Bot Desktop: a headless Xfce screen per profile on the gateway host (Linux), streamed to Hermes + # Desktop where a human can watch, take over (logins, 2FA, CAPTCHAs) and hand back. `hermes desktop`. + "bot_desktop": { + "geometry": "1440x900", + # Start the screen automatically the first time computer_use or a headed browser needs a display. + "auto_start": True, + }, "computer_use": { # cua-driver's upstream PostHog telemetry defaults ON; Hermes sets # CUA_DRIVER_RS_TELEMETRY_ENABLED=0 in every child env unless this is true. diff --git a/hermes_cli/dashboard_auth/ws_tickets.py b/hermes_cli/dashboard_auth/ws_tickets.py index aef1456b84e1..65391ae239f8 100644 --- a/hermes_cli/dashboard_auth/ws_tickets.py +++ b/hermes_cli/dashboard_auth/ws_tickets.py @@ -33,11 +33,13 @@ class TicketInvalid(Exception): """Ticket missing, expired, or already consumed.""" -def mint_ticket(*, user_id: str, provider: str) -> str: +def mint_ticket(*, user_id: str, provider: str, extra: Optional[Dict[str, Any]] = None) -> str: """One-shot base64url ticket (32 random bytes) bound to this identity; ``consume_ticket`` - hands the ``info`` dict back to the WS handler.""" + hands the ``info`` dict back to the WS handler. ``extra`` rides along for routes that need + server-chosen context (the Bot Desktop bridge pins the RFB socket's profile home here so a + client can never pick another profile's screen).""" ticket = secrets.token_urlsafe(32) - info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time())} + info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time()), **(extra or {})} with _lock: _tickets[ticket] = (int(time.time()) + TTL_SECONDS, info) _gc_expired_locked() diff --git a/hermes_cli/subcommands/computer_use.py b/hermes_cli/subcommands/computer_use.py index ac0d52f4b88a..22e0effcd6da 100644 --- a/hermes_cli/subcommands/computer_use.py +++ b/hermes_cli/subcommands/computer_use.py @@ -174,6 +174,12 @@ def build_computer_use_parser(subparsers) -> None: "grant", help="Request the grants (opens the dialog attributed to CuaDriver)") _perms_actions = {"grant": _cu_perms_grant, "status": _cu_perms_status} + from hermes_cli.subcommands.computer_use_screen import build_screen_parser + build_screen_parser(computer_use_sub, add_json_flag) + + def _cu_screen(args): + return args.screen_func(args) + def _cu_permissions(args): handler = _perms_actions.get(getattr(args, "computer_use_perms_action", None)) if handler is not None: @@ -181,7 +187,7 @@ def _cu_permissions(args): computer_use_perms.print_help() _actions = {"install": _cu_install, "status": _cu_status, "doctor": _cu_doctor, - "permissions": _cu_permissions} + "permissions": _cu_permissions, "screen": _cu_screen} def cmd_computer_use(args): handler = _actions.get(getattr(args, "computer_use_action", None)) diff --git a/hermes_cli/subcommands/computer_use_screen.py b/hermes_cli/subcommands/computer_use_screen.py new file mode 100644 index 000000000000..5c2dc5260638 --- /dev/null +++ b/hermes_cli/subcommands/computer_use_screen.py @@ -0,0 +1,108 @@ +"""``hermes computer-use screen`` — the Bot Desktop screen a profile's ``computer_use`` drives on a +headless Linux gateway host, viewable from Hermes Desktop. ``status`` / ``start`` / ``stop`` / +``install`` mirror the Desktop pane's controls for ops shells and cloud images.""" + +from __future__ import annotations + +import json +import subprocess +import sys + + +def _screen_status(args) -> int: + from tools.bot_desktop import lease, runtime + st = runtime.status() + if bool(getattr(args, "json", False)): + print(json.dumps({**st.as_dict(), "lease": lease.get().as_dict()}, indent=2, sort_keys=True)) + return 0 if st.running else 1 + if not st.supported: + print("Bot Desktop screens run on Linux gateway hosts only (this host keeps its real display).") + return 1 + if not st.installed: + print("Bot Desktop: packages missing → " + ", ".join(st.missing)) + print(" Install: " + (st.install_command or "hermes computer-use screen install")) + return 1 + if st.running: + holder = lease.get() + who = f"human ({holder.viewer_id})" if holder.holder == "human" else "agent" + print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry}), pid {st.pid}") + print(f" control: {who} rfb socket: {st.socket}") + print(" View it: Hermes Desktop → Bots → this bot → Screen") + return 0 + print(f"Bot Desktop [{st.profile}]: installed, not running. Start: hermes computer-use screen start") + return 1 + + +def _screen_start(args) -> int: + from tools.bot_desktop import runtime + try: + st = runtime.start() + except RuntimeError as exc: + print(f"Bot Desktop: {exc}") + return 1 + print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry})") + return 0 + + +def _screen_stop(args) -> int: + from tools.bot_desktop import runtime + print("Bot Desktop: stopped" if runtime.stop() else "Bot Desktop: was not running") + return 0 + + +def _screen_install(args) -> int: + from tools.bot_desktop import runtime + if not runtime.is_supported_host(): + print("Bot Desktop screens run on Linux gateway hosts only.") + return 1 + if not runtime.missing_binaries(): + print("Bot Desktop: packages already installed.") + return 0 + cmd = runtime.install_command() + if cmd is None: + print("Bot Desktop: no supported package manager (apt/dnf/pacman) found. Install TigerVNC (Xvnc) and " + "the Xfce core (xfwm4, xfce4-panel, xfdesktop, xfce4-settings) by hand.") + return 1 + print(f"Bot Desktop: installing → {cmd}") + if not bool(getattr(args, "yes", False)) and sys.stdin.isatty(): + answer = input("Proceed? [Y/n] ").strip().lower() + if answer not in ("", "y", "yes"): + return 1 + rc = subprocess.run(cmd, shell=True, stdin=None).returncode # windows-footgun: ok — Linux-only, apt/dnf/pacman + if rc != 0: + print(f"Bot Desktop: installer exited {rc}") + return rc + missing = runtime.missing_binaries() + if missing: + print("Bot Desktop: still missing " + ", ".join(missing)) + return 1 + print("Bot Desktop: ready. Start with `hermes computer-use screen start` or from Hermes Desktop.") + return 0 + + +SCREEN_ACTIONS = {"status": _screen_status, "start": _screen_start, "stop": _screen_stop, "install": _screen_install} + + +def build_screen_parser(computer_use_sub, add_json_flag) -> None: + screen = computer_use_sub.add_parser( + "screen", help="Bot Desktop: the headless screen this profile's computer_use drives (Linux)", + description="On a headless Linux gateway host Hermes gives each profile its own Xfce screen\n" + "(TigerVNC Xvnc on a private Unix socket). The agent's computer_use and headed\n" + "browser act on it; Hermes Desktop shows it live and lets a human take over for\n" + "logins, 2FA or CAPTCHAs, then hand control back.\n\n" + "`install` adds the system packages (apt/dnf/pacman); `start`/`stop` manage this\n" + "profile's screen; `status` shows display, control holder and socket.") + sub = screen.add_subparsers(dest="computer_use_screen_action") + st = sub.add_parser("status", help="Show whether this profile's screen is installed/running and who holds control") + add_json_flag(st, "Emit the status payload as JSON.") + sub.add_parser("start", help="Start this profile's screen") + sub.add_parser("stop", help="Stop this profile's screen (hands control back to the agent first)") + inst = sub.add_parser("install", help="Install TigerVNC + Xfce core via the host package manager") + inst.add_argument("-y", "--yes", action="store_true", help="Do not ask before running the package manager") + + def _cmd(args): + handler = SCREEN_ACTIONS.get(str(getattr(args, "computer_use_screen_action", None) or "")) + if handler is not None: + return handler(args) + screen.print_help() + screen.set_defaults(screen_func=_cmd) diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py new file mode 100644 index 000000000000..a9fe07a5c3e5 --- /dev/null +++ b/hermes_cli/web_routers/display.py @@ -0,0 +1,144 @@ +"""``/api/display/ws`` — raw RFB over WebSocket for the Bot Desktop viewer. + +The Desktop renderer calls ``display.observe`` on its authenticated ``/api/ws`` connection, gets a +single-use 30 s ticket pinned to that profile's RFB socket, then opens this route with +``?display_ticket=``. No websockify, no new port: the bridge splices the profile's 0600 Unix socket +into the WebSocket as binary frames with backpressure both ways, and runs the client stream through +:class:`tools.bot_desktop.rfb_filter.RfbClientFilter` so keyboard, pointer and clipboard reach Xvnc +only from the viewer that currently holds the lease. noVNC's ``viewOnly`` is UX; this is the gate. + +A lease change closes the evicted viewer's socket with 4000 ``control-taken`` so its UI drops back to +Watch mode and reconnects. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Optional + +from fastapi import APIRouter, WebSocket, WebSocketDisconnect + +from hermes_cli.web_server_chat import _ws_request_is_allowed + +_log = logging.getLogger(__name__) +router = APIRouter() + +_READ_CHUNK = 64 * 1024 +_CLOSE_CONTROL_TAKEN = 4000 +_CLOSE_DESKTOP_GONE = 4001 +_CLOSE_BAD_TICKET = 4401 +_CLOSE_NOT_ALLOWED = 4403 +_CLOSE_PROTOCOL = 1003 + + +def _consume_display_ticket(ws: WebSocket) -> Optional[dict]: + from hermes_cli.dashboard_auth.ws_tickets import TicketInvalid, consume_ticket + ticket = ws.query_params.get("display_ticket", "") + if not ticket: + return None + try: + info = consume_ticket(ticket) + except TicketInvalid: + return None + if info.get("provider") != "bot-desktop" or not info.get("hermes_home"): + return None + return info + + +@router.websocket("/api/display/ws") +async def display_ws(ws: WebSocket) -> None: + if not _ws_request_is_allowed(ws): + await ws.close(code=_CLOSE_NOT_ALLOWED) + return + info = _consume_display_ticket(ws) + if info is None: + await ws.close(code=_CLOSE_BAD_TICKET, reason="display ticket missing, expired or used") + return + + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _lease + from tools.bot_desktop.rfb_filter import RfbClientFilter + from pathlib import Path + + sock = Path(info["hermes_home"]) / "bot-desktop" / "rfb.sock" + profile_key = hermes_home_key(info["hermes_home"]) + viewer_id = str(info.get("viewer_id") or info.get("user_id") or "viewer") + if not sock.exists(): + await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop is not running") + return + try: + reader, writer = await asyncio.open_unix_connection(str(sock)) + except OSError as exc: + _log.warning("display ws: cannot reach RFB socket %s: %s", sock, exc) + await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop socket unreachable") + return + + await ws.accept() + loop = asyncio.get_running_loop() + evicted = asyncio.Event() + held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_key)} + + def _on_lease(key: str, lease) -> None: + # A viewer that held control during this connection and lost it to ANOTHER human is kicked so + # its UI repaints; a plain hand-back to the agent, pure watchers and the new holder stay connected. + if key != profile_key: + return + mine = lease.holder == _lease.HUMAN and lease.viewer_id == viewer_id + if mine: + held["ever"] = True + elif held["ever"] and lease.holder == _lease.HUMAN: + loop.call_soon_threadsafe(evicted.set) + unsubscribe = _lease.on_change(_on_lease) + + rfb_filter = RfbClientFilter(lambda: _lease.viewer_may_send_input(viewer_id, profile_key=profile_key)) + + async def rfb_to_ws() -> None: + while True: + chunk = await reader.read(_READ_CHUNK) + if not chunk: + return + await ws.send_bytes(chunk) # awaiting the send is the backpressure toward Xvnc + + async def ws_to_rfb() -> None: + while True: + message = await ws.receive() + if message.get("type") == "websocket.disconnect": + return + data = message.get("bytes") + if data is None: + await ws.close(code=_CLOSE_PROTOCOL, reason="RFB is binary") + return + try: + allowed = rfb_filter.feed(data) + except ValueError as exc: + await ws.close(code=_CLOSE_PROTOCOL, reason=str(exc)[:100]) + return + if allowed: + writer.write(allowed) + await writer.drain() # backpressure toward the browser + + async def watch_eviction() -> None: + await evicted.wait() + await ws.close(code=_CLOSE_CONTROL_TAKEN, reason="control-taken") + + tasks = [asyncio.create_task(rfb_to_ws()), asyncio.create_task(ws_to_rfb()), + asyncio.create_task(watch_eviction())] + try: + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for t in pending: + t.cancel() + for t in done: + exc = t.exception() + if exc and not isinstance(exc, (WebSocketDisconnect, ConnectionError)): + _log.debug("display ws ended: %r", exc) + finally: + unsubscribe() + writer.close() + # Closing the viewer window hands control back; a stale holder never pins the agent out. + if _lease.viewer_may_send_input(viewer_id, profile_key=profile_key): + _lease.release(viewer_id, profile_key=profile_key) + try: + await ws.close() + except Exception: # already closed by the peer or by an eviction + pass diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index d95f8182a696..73faef43e6b3 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -932,6 +932,7 @@ def _get_dashboard_plugins(force_rescan: bool = False) -> list: status as _status_routes, actions as _actions_routes, audio as _audio_routes, + display as _display_routes, sessions as _sessions_routes, profiles as _profiles_routes, memory_providers as _memory_providers_routes, @@ -955,6 +956,7 @@ def _get_dashboard_plugins(force_rescan: bool = False) -> list: app.include_router(_status_routes.router) app.include_router(_actions_routes.router) app.include_router(_audio_routes.router) +app.include_router(_display_routes.router) app.include_router(_actions_routes.status_router) app.include_router(_sessions_routes.list_router) app.include_router(_profiles_routes.sessions_router) diff --git a/package-lock.json b/package-lock.json index eaf154d45076..a24c4496ef6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ }, "apps/bootstrap-installer": { "name": "@hermes/bootstrap-installer", - "version": "0.0.1", + "version": "0.21.1", "dependencies": { "@nous-research/ui": "0.18.2", "@tailwindcss/typography": "0.5.20", @@ -85,6 +85,7 @@ "@lezer/highlight": "1.2.3", "@nanostores/react": "1.1.0", "@nous-research/ui": "0.18.2", + "@novnc/novnc": "1.7.0", "@streamdown/code": "1.1.1", "@streamdown/math": "1.0.2", "@tabler/icons-react": "3.44.0", @@ -3498,6 +3499,12 @@ } } }, + "node_modules/@novnc/novnc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@novnc/novnc/-/novnc-1.7.0.tgz", + "integrity": "sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==", + "license": "MPL-2.0" + }, "node_modules/@npmcli/agent": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.2.2.tgz", diff --git a/tests/hermes_cli/test_display_ws_ticket.py b/tests/hermes_cli/test_display_ws_ticket.py new file mode 100644 index 000000000000..fc072596b15f --- /dev/null +++ b/tests/hermes_cli/test_display_ws_ticket.py @@ -0,0 +1,27 @@ +"""The display bridge admits only a display ticket minted for THIS profile's socket: a gateway ticket, +an expired ticket, or one for another provider is refused before any socket is dialled.""" + +from __future__ import annotations + +from hermes_cli.dashboard_auth import ws_tickets +from hermes_cli.web_routers import display + + +class _Ws: + def __init__(self, **params): + self.query_params = params + + +def test_display_ticket_must_be_a_bot_desktop_ticket_pinned_to_a_profile_home(monkeypatch): + ws_tickets._reset_for_tests() + gateway_ticket = ws_tickets.mint_ticket(user_id="u", provider="google") + assert display._consume_display_ticket(_Ws(display_ticket=gateway_ticket)) is None + + unpinned = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop") + assert display._consume_display_ticket(_Ws(display_ticket=unpinned)) is None + + good = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop", + extra={"hermes_home": "/srv/hermes/bot-a", "viewer_id": "v"}) + info = display._consume_display_ticket(_Ws(display_ticket=good)) + assert info and info["hermes_home"] == "/srv/hermes/bot-a" and info["viewer_id"] == "v" + assert display._consume_display_ticket(_Ws(display_ticket=good)) is None, "single use" diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index f5c2c431fd29..34811b799b72 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -35,6 +35,24 @@ def _no_host_browser_use_cli(): yield +@pytest.fixture(autouse=True) +def _no_host_bot_desktop_autostart(): + """Keep the host's TigerVNC/Xfce install out of tests. + + ``computer_use`` auto-starts the profile's Bot Desktop on a headless Linux + host with the packages installed, so a developer box that has them would + launch a real Xvnc + Xfce session per test. Pin the binaries to "missing"; + tests that exercise the desktop path monkeypatch ``runtime`` themselves. + """ + try: + from tools.bot_desktop import runtime as bd_runtime + except Exception: + yield + return + with patch.object(bd_runtime, "missing_binaries", lambda: ["Xvnc"]): + yield + + @pytest.fixture(autouse=True) def _materialize_mcp_sdk_symbols(): """Materialize the lazily-imported MCP SDK before each tools test. diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py new file mode 100644 index 000000000000..9266ad1ec261 --- /dev/null +++ b/tests/tools/test_bot_desktop_lease.py @@ -0,0 +1,67 @@ +"""Bot Desktop invariants: the byte-level RFB input gate follows the lease, and computer_use refuses +every action (capture included) while a human holds the screen.""" + +from __future__ import annotations + +import json + +import pytest + +from tools.bot_desktop import lease +from tools.bot_desktop.rfb_filter import RfbClientFilter + +_HANDSHAKE = b"RFB 003.008\n" + b"\x01" + b"\x00" +_KEY = b"\x04\x01\x00\x00\x00\x00\x00\x61" # KeyEvent 'a' down +# QEMU Extended KeyEvent (type 255, sub 0): what noVNC sends once Xvnc advertises the pseudo-encoding. +_QEMU_KEY = bytes([255, 0, 0, 1]) + (0x65).to_bytes(4, "big") + (0x12).to_bytes(4, "big") +_POINTER = b"\x05\x01\x00\x10\x00\x10" # PointerEvent, button 1 +_CUT = b"\x06\x00\x00\x00\x00\x00\x00\x02hi" # ClientCutText "hi" +_FBUR = b"\x03\x00" + b"\x00" * 8 # FramebufferUpdateRequest +_SETENC = b"\x02\x00\x00\x02" + b"\x00\x00\x00\x07" + b"\xff\xff\xff\x21" # SetEncodings x2 + + +@pytest.fixture(autouse=True) +def _fresh_lease(): + lease._reset_for_tests() + yield + lease._reset_for_tests() + + +def test_rfb_filter_forwards_input_only_from_the_lease_holder_across_arbitrary_chunking(): + f = RfbClientFilter(lambda: lease.viewer_may_send_input("v1")) + head = f.feed(_HANDSHAKE) + assert head[-1:] == b"\x01", "ClientInit is forced shared so a viewer never kicks the agent's watcher" + + # Agent holds: read-only messages pass, input is dropped, even when split byte by byte. + stream = _KEY + _FBUR + _POINTER + _SETENC + _CUT + _QEMU_KEY + out = b"".join(f.feed(stream[i:i + 1]) for i in range(len(stream))) + assert out == _FBUR + _SETENC + + lease.acquire("v1") + assert f.feed(_KEY + _POINTER + _QEMU_KEY) == _KEY + _POINTER + _QEMU_KEY + + lease.acquire("v2") # last writer wins: v1 is evicted from input on the very next message + assert f.feed(_KEY) == b"" + assert lease.release("v1").holder == lease.HUMAN, "a stale viewer's release must not yank control from v2" + assert lease.release("v2").holder == lease.AGENT + + +def test_computer_use_refuses_every_action_while_a_human_holds_the_screen(monkeypatch): + from tools.computer_use import tool + + calls = [] + monkeypatch.setattr(tool, "_get_backend", lambda session_id="": calls.append(session_id) or object()) + lease.acquire("human") + for action in ("capture", "click", "type", "list_windows"): + res = json.loads(tool.handle_computer_use({"action": action, "text": "pw"})) + assert res["code"] == "human_has_control", action + assert calls == [], "the driver is never touched while the human may be typing a credential" + + # Handoff round trip: the agent asks, the human takes over and hands back, the agent is unblocked. + asked = json.loads(tool.handle_computer_use({"action": "request_handoff", "reason": "log in"})) + assert asked["ok"] and asked["state"]["pending_handoff"] == "log in" + lease.acquire("human", reason="log in") + assert lease.get().pending_handoff is None + lease.release("human") + done = json.loads(tool.handle_computer_use({"action": "wait_for_human", "seconds": 1})) + assert done["ok"] and done["state"]["holder"] == lease.AGENT diff --git a/tools/bot_desktop/__init__.py b/tools/bot_desktop/__init__.py new file mode 100644 index 000000000000..2ffa400fd64e --- /dev/null +++ b/tools/bot_desktop/__init__.py @@ -0,0 +1,5 @@ +"""Bot Desktop: per-profile headless Xfce desktop over RFB, viewed from Hermes Desktop. + +``runtime`` owns the Xvnc/Xfce process and the published env; ``lease`` owns who may drive the +screen (agent vs. human); ``rfb_filter`` is the byte-level input gate the WebSocket bridge applies. +""" diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh new file mode 100755 index 000000000000..60c3635982c7 --- /dev/null +++ b/tools/bot_desktop/launcher.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Hermes Bot Desktop — one headless Xfce desktop per Hermes profile, served over RFB. +# +# Spawned by tools/bot_desktop/runtime.py with HERMES_BD_* variables set. Runs TigerVNC's Xvnc +# (X server + RFB server in one process; damage-driven, resizable via SetDesktopSize) listening on a +# 0600 Unix socket only, then a minimal Xfce started component-wise under a private dbus session. +# +# Why not startxfce4 / xfce4-session: xfce4-session expects a logind session scope; outside one it +# spawns polkit agents that pop empty dialogs and light-locker/xfce4-screensaver lock the desktop for a +# user who has no password. Starting xfsettingsd -> xfwm4 -> xfdesktop -> xfce4-panel directly, with +# the screensaver/locker/power-manager autostarts masked, is the shape every headless-VNC recipe +# converges on (TigerVNC #1096/#581, OpenOnDemand's apptainer desktop, the Arch wiki). +# +# Why not the xfce4 metapackage: it drags in the screensaver, power manager and polkit agent this +# script exists to keep out. +set -euo pipefail + +: "${HERMES_BD_PROFILE:?}" # profile name (display name in the VNC title) +: "${HERMES_BD_DISPLAY_NUM:?}" # allocated by runtime.py +: "${HERMES_BD_SOCKET:?}" # RFB unix socket path +: "${HERMES_BD_XAUTH:?}" # Xauthority path +: "${HERMES_BD_ENV_FILE:?}" # where to publish DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS +: "${HERMES_BD_CONFIG_HOME:?}" # per-profile XDG_CONFIG_HOME (xfconf lives here) +GEOM="${HERMES_BD_GEOMETRY:-1440x900}" +DEPTH=24 + +export XDG_CONFIG_HOME="$HERMES_BD_CONFIG_HOME" +export XDG_CACHE_HOME="${HERMES_BD_CACHE_HOME:-$HERMES_BD_CONFIG_HOME/.cache}" +export XDG_DATA_HOME="${HERMES_BD_DATA_HOME:-$HERMES_BD_CONFIG_HOME/.local-share}" +export XDG_SESSION_TYPE=x11 XDG_CURRENT_DESKTOP=XFCE +export GDK_BACKEND=x11 QT_QPA_PLATFORM=xcb NO_AT_BRIDGE=1 GTK_A11Y=none +export LANG="${LANG:-C.UTF-8}" +# Inheriting a login session's bus/session manager yields "Another session manager is already +# running" / "Unable to contact settings server". +unset SESSION_MANAGER DBUS_SESSION_BUS_ADDRESS DISPLAY XAUTHORITY WAYLAND_DISPLAY + +mkdir -p "$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" "$XDG_CONFIG_HOME/autostart" \ + "$XDG_CACHE_HOME" "$XDG_DATA_HOME" "$(dirname "$HERMES_BD_SOCKET")" + +export DISPLAY=":$HERMES_BD_DISPLAY_NUM" +export XAUTHORITY="$HERMES_BD_XAUTH" + +# Stale lock files from a crashed server block restart. +rm -f "$HERMES_BD_SOCKET" "/tmp/.X${HERMES_BD_DISPLAY_NUM}-lock" "/tmp/.X11-unix/X${HERMES_BD_DISPLAY_NUM}" +: > "$XAUTHORITY"; chmod 600 "$XAUTHORITY" +xauth -q -f "$XAUTHORITY" add "$DISPLAY" MIT-MAGIC-COOKIE-1 "$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" + +# ---- pre-seed xfconf BEFORE xfconfd starts (it caches; edits after start are overwritten) ---- +X="$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" +[[ -e "$X/xfwm4.xml" ]] || cat > "$X/xfwm4.xml" <<'EOF' + + + + + + + + +EOF +[[ -e "$X/xfce4-screensaver.xml" ]] || cat > "$X/xfce4-screensaver.xml" <<'EOF' + + + + + +EOF +[[ -e "$X/xsettings.xml" ]] || cat > "$X/xsettings.xml" <<'EOF' + + + + + + + + + + + +EOF +[[ -e "$X/xfce4-desktop.xml" ]] || cat > "$X/xfce4-desktop.xml" <<'EOF' + + + + + + + + + + + + + + + + + + + + +EOF +# The vendor default panel layout suppresses the first-run "Welcome to the panel" dialog. +if [[ ! -e "$X/xfce4-panel.xml" ]]; then + for d in /etc/xdg/xfce4/panel/default.xml /usr/share/xfce4-panel/default.xml \ + /etc/xdg/xdg-xubuntu/xfce4/panel/default.xml; do + [[ -e "$d" ]] && { cp "$d" "$X/xfce4-panel.xml"; break; } + done +fi +# Mask system autostarts that want logind/polkit/keyring/at-spi. +for a in xfce4-screensaver light-locker xfce4-power-manager xfce-polkit \ + polkit-gnome-authentication-agent-1 lxpolkit xfce4-notifyd blueman at-spi-dbus-bus \ + gnome-keyring-pkcs11 gnome-keyring-secrets gnome-keyring-ssh xdg-user-dirs; do + [[ -e "$XDG_CONFIG_HOME/autostart/$a.desktop" ]] || \ + printf '[Desktop Entry]\nType=Application\nName=%s\nHidden=true\n' "$a" > "$XDG_CONFIG_HOME/autostart/$a.desktop" +done + +# ---- X server + RFB (TigerVNC Xvnc), Unix socket only ---- +# SecurityTypes None is safe ONLY because -rfbport -1 disables TCP and the 0600 socket is reachable +# solely by the gateway process, whose WebSocket bridge performs the real authentication. +Xvnc "$DISPLAY" -geometry "$GEOM" -depth "$DEPTH" -dpi 96 \ + -rfbport -1 -rfbunixpath "$HERMES_BD_SOCKET" -rfbunixmode 0600 \ + -SecurityTypes None -AlwaysShared -AcceptSetDesktopSize -FrameRate 30 \ + -desktop "hermes:$HERMES_BD_PROFILE" -auth "$XAUTHORITY" -nolisten tcp \ + -Log '*:stderr:30' & +XVNC_PID=$! +trap 'kill "$XVNC_PID" 2>/dev/null || true' EXIT +for _ in $(seq 1 100); do + xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 && break + kill -0 "$XVNC_PID" 2>/dev/null || { echo "Xvnc exited during startup" >&2; exit 1; } + sleep 0.1 +done +xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 || { echo "Xvnc did not become ready" >&2; exit 1; } + +setxkbmap -display "$DISPLAY" us 2>/dev/null || true # RFB keysyms + xdotool assume a known layout +xsetroot -display "$DISPLAY" -solid '#1c1f29' 2>/dev/null || true +xset -display "$DISPLAY" s off -dpms s noblank 2>/dev/null || true + +# ---- private session bus + Xfce components (no xfce4-session) ---- +# dbus-run-session scopes the bus to this subshell: no leaked dbus-daemons on restart. The env file +# is written from INSIDE the bus so DBUS_SESSION_BUS_ADDRESS is the real one; runtime.py and every +# cua-driver / browser spawn for this profile source it. +exec dbus-run-session -- bash -c ' + set -e + umask 077 + printf "DISPLAY=%s\nXAUTHORITY=%s\nDBUS_SESSION_BUS_ADDRESS=%s\nXDG_CONFIG_HOME=%s\nXDG_CACHE_HOME=%s\nXDG_DATA_HOME=%s\n" \ + "$DISPLAY" "$XAUTHORITY" "$DBUS_SESSION_BUS_ADDRESS" "$XDG_CONFIG_HOME" "$XDG_CACHE_HOME" "$XDG_DATA_HOME" \ + > "$HERMES_BD_ENV_FILE.tmp" && mv -f "$HERMES_BD_ENV_FILE.tmp" "$HERMES_BD_ENV_FILE" + xfsettingsd --sm-client-disable --daemon 2>/dev/null || true + xfwm4 --compositor=off --sm-client-disable & + for _ in $(seq 1 50); do xprop -root _NET_SUPPORTING_WM_CHECK >/dev/null 2>&1 && break; sleep 0.1; done + xfdesktop --sm-client-disable --disable-wm-check & + exec xfce4-panel --sm-client-disable --disable-wm-check +' diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py new file mode 100644 index 000000000000..405c68393650 --- /dev/null +++ b/tools/bot_desktop/lease.py @@ -0,0 +1,154 @@ +"""Who may drive a profile's Bot Desktop screen: the agent (default) or exactly one human viewer. + +The lease is the single truth shared by the RFB bridge (drops human input from non-holders), the +``computer_use`` tool (refuses to act while a human holds control — the person may be typing a +credential, so even screenshots are refused; fail closed rather than trusting the agent to pause +itself) and the Desktop UI (Watch / Take over / Hand back). + +Per-process, keyed by ``hermes_home_key()`` so multiplexed profiles never share a lease. Handoff +requests raised by the agent (``request_handoff``) are how it asks for hands and later learns the +human is done: ``wait_for_release`` blocks the tool call until the lease returns to the agent. +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +from hermes_constants import hermes_home_key + +AGENT = "agent" +HUMAN = "human" + + +class HumanHasControl(RuntimeError): + """Raised by screen-driving tools while a human holds the lease.""" + + +@dataclass +class Lease: + holder: str = AGENT + viewer_id: Optional[str] = None + since: float = field(default_factory=time.time) + reason: str = "" + pending_handoff: Optional[str] = None # agent's reason for asking, until the human takes over + + def as_dict(self) -> Dict[str, object]: + return {"holder": self.holder, "viewer_id": self.viewer_id, "since": self.since, + "reason": self.reason, "pending_handoff": self.pending_handoff} + + +_lock = threading.Condition() +_leases: Dict[str, Lease] = {} +_listeners: List[Callable[[str, Lease], None]] = [] + + +def _key(profile_key: Optional[str]) -> str: + return profile_key or hermes_home_key() + + +def get(profile_key: Optional[str] = None) -> Lease: + with _lock: + return _leases.setdefault(_key(profile_key), Lease()) + + +def on_change(listener: Callable[[str, Lease], None]) -> Callable[[], None]: + """Subscribe to lease transitions (gateway broadcasts them to Desktop clients).""" + with _lock: + _listeners.append(listener) + + def _off() -> None: + with _lock: + if listener in _listeners: + _listeners.remove(listener) + return _off + + +def _notify(key: str, lease: Lease) -> None: + for cb in list(_listeners): + try: + cb(key, lease) + except Exception: # a broken subscriber must not wedge the handoff + pass + + +def acquire(viewer_id: str, *, profile_key: Optional[str] = None, reason: str = "") -> Lease: + """Human ``viewer_id`` takes control. Last writer wins: a second viewer evicts the first, and the + RFB bridge closes the evicted socket so its UI drops to view-only.""" + key = _key(profile_key) + with _lock: + lease = _leases.setdefault(key, Lease()) + lease.holder, lease.viewer_id, lease.since, lease.reason = HUMAN, viewer_id, time.time(), reason + lease.pending_handoff = None + _lock.notify_all() + _notify(key, lease) + return lease + + +def release(viewer_id: Optional[str] = None, *, profile_key: Optional[str] = None) -> Lease: + """Return control to the agent. With ``viewer_id`` only that holder may release (a stale viewer + closing its window must not yank control from the one who took over after it).""" + key = _key(profile_key) + with _lock: + lease = _leases.setdefault(key, Lease()) + if viewer_id is not None and lease.holder == HUMAN and lease.viewer_id != viewer_id: + return lease + lease.holder, lease.viewer_id, lease.since, lease.reason = AGENT, None, time.time(), "" + lease.pending_handoff = None # "hand back" answers an open request even if nobody formally took over + _lock.notify_all() + _notify(key, lease) + return lease + + +def request_handoff(reason: str, *, profile_key: Optional[str] = None) -> Lease: + """Agent asks a human to take over (login, 2FA, CAPTCHA, payment). Recorded so the UI can show + why and the bridge can page the user; control itself still flips only on ``acquire``.""" + key = _key(profile_key) + with _lock: + lease = _leases.setdefault(key, Lease()) + lease.pending_handoff = reason + _lock.notify_all() + _notify(key, lease) + return lease + + +def wait_for_release(*, timeout: float, profile_key: Optional[str] = None) -> bool: + """Block until the agent holds the lease (and no handoff is pending) or ``timeout`` elapses. + True when control is back with the agent.""" + key = _key(profile_key) + deadline = time.monotonic() + timeout + with _lock: + while True: + lease = _leases.setdefault(key, Lease()) + if lease.holder == AGENT and lease.pending_handoff is None: + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + _lock.wait(remaining) + + +def human_holds(profile_key: Optional[str] = None) -> bool: + return get(profile_key).holder == HUMAN + + +def viewer_may_send_input(viewer_id: str, *, profile_key: Optional[str] = None) -> bool: + lease = get(profile_key) + return lease.holder == HUMAN and lease.viewer_id == viewer_id + + +def assert_agent_may_act(profile_key: Optional[str] = None) -> None: + lease = get(profile_key) + if lease.holder == HUMAN: + raise HumanHasControl( + "A human has taken over this desktop (they may be entering a credential). Screen actions and " + "captures are refused until they hand control back; call computer_use action='wait_for_human' " + "to block until then.") + + +def _reset_for_tests() -> None: + with _lock: + _leases.clear() + _listeners.clear() diff --git a/tools/bot_desktop/rfb_filter.py b/tools/bot_desktop/rfb_filter.py new file mode 100644 index 000000000000..804ad9d3fd42 --- /dev/null +++ b/tools/bot_desktop/rfb_filter.py @@ -0,0 +1,94 @@ +"""Byte-level RFB client→server gate for the Bot Desktop WebSocket bridge. + +noVNC's ``viewOnly`` is a UI hint; anyone holding the socket could still inject input. The bridge +parses the client stream and forwards only non-input messages from viewers that do not hold the +lease. RFB messages do not align with WebSocket frames, so this is a stateful stream parser fed +arbitrary chunks (RFC 6143 §7.5 layouts; TigerVNC's EnableContinuousUpdates 150 and Fence 248 pass +through untouched since they carry no input; QEMU Extended KeyEvent 255 is keyboard input — noVNC +switches to it as soon as Xvnc advertises the pseudo-encoding — so it is gated like KeyEvent). + +Xvnc runs ``-SecurityTypes None``, so the handshake is fixed-size: 12-byte version, 1-byte security +choice, then ``ClientInit`` (1 byte). ``ServerInit`` is server→client and never crosses this filter. +""" + +from __future__ import annotations + +from typing import Callable + +_INPUT_TYPES = {4, 5, 6, 255} # KeyEvent, PointerEvent, ClientCutText, QEMU Extended KeyEvent + +# Fixed-length client messages: type -> total length including the type byte. +_FIXED = { + 0: 20, # SetPixelFormat + 3: 10, # FramebufferUpdateRequest + 4: 8, # KeyEvent + 5: 6, # PointerEvent + 150: 10, # EnableContinuousUpdates + 255: 12, # QEMU client message; sub-type 0 = Extended KeyEvent (the only one noVNC sends) +} +_SET_ENCODINGS = 2 +_CLIENT_CUT_TEXT = 6 +_FENCE = 248 + + +class RfbClientFilter: + """Feed client bytes with :meth:`feed`; get back the bytes allowed to reach Xvnc. + + ``allow_input`` is consulted per message so a lease flip mid-stream applies to the very next + key or pointer event. + """ + + def __init__(self, allow_input: Callable[[], bool]) -> None: + self._allow_input = allow_input + self._buf = bytearray() + self._handshake_left = 12 + 1 + 1 # version + security type + ClientInit(shared flag) + + def feed(self, chunk: bytes) -> bytes: + self._buf += chunk + out = bytearray() + if self._handshake_left: + take = min(self._handshake_left, len(self._buf)) + if take: + head = bytes(self._buf[:take]) + # ClientInit shared-flag: force shared so a human viewer never disconnects the agent's + # watcher or another observer (Xvnc also runs -AlwaysShared; belt and braces at zero cost). + if self._handshake_left - take == 0 and take >= 1: + head = head[:-1] + b"\x01" + out += head + del self._buf[:take] + self._handshake_left -= take + if self._handshake_left: + return bytes(out) + while self._buf: + length = self._message_length() + if length is None or len(self._buf) < length: + break + msg = bytes(self._buf[:length]) + del self._buf[:length] + if msg[0] in _INPUT_TYPES and not self._allow_input(): + continue + out += msg + return bytes(out) + + def _message_length(self) -> int | None: + t = self._buf[0] + if t in _FIXED: + return _FIXED[t] + if t == _SET_ENCODINGS: + if len(self._buf) < 4: + return None + n = int.from_bytes(self._buf[2:4], "big") + return 4 + 4 * n + if t == _CLIENT_CUT_TEXT: + if len(self._buf) < 8: + return None + n = int.from_bytes(self._buf[4:8], "big", signed=True) + # Extended clipboard (RFB 3.8 + TigerVNC): negative length, |n| bytes follow. + return 8 + abs(n) + if t == _FENCE: + if len(self._buf) < 9: + return None + return 9 + self._buf[8] + # Unknown client message: we cannot frame it, and forwarding blind would let an input message + # hide behind it. Drop the rest of the stream; the viewer reconnects. + raise ValueError(f"unknown RFB client message type {t}") diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py new file mode 100644 index 000000000000..cead59dd065d --- /dev/null +++ b/tools/bot_desktop/runtime.py @@ -0,0 +1,295 @@ +"""Bot Desktop runtime: one headless Xfce desktop per Hermes profile, served over RFB on a private +Unix socket, viewed and driven from Hermes Desktop. + +Layout under ``/bot-desktop/``: ``display`` (allocated X display number), ``rfb.sock`` +(Xvnc RFB Unix socket, 0600), ``Xauthority``, ``env`` (DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS +published by the launcher once Xfce's bus exists), ``launcher.pid``, ``launcher.log``, ``xdg/`` +(per-profile XDG_CONFIG_HOME so two profiles never share xfconf). Everything is profile-scoped via +``get_hermes_home()`` so N profiles in one gateway get N desktops: one screen per bot on the shared +machine. + +The launcher is ``launcher.sh`` next to this module; :func:`desktop_env` is what cua-driver and headed +Chromium spawns merge in so the agent acts on this profile's screen and nowhere else. +""" + +from __future__ import annotations + +import logging +import os +import shutil +import signal +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Optional + +from hermes_constants import get_hermes_home + +logger = logging.getLogger(__name__) + +_LAUNCHER = Path(__file__).with_name("launcher.sh") + +# Display numbers below 10 collide with real seats and default Xvfb recipes (:99 is popular too); scan a +# private band and record the choice so restarts reuse it. +_DISPLAY_MIN, _DISPLAY_MAX = 20, 89 + +# Binaries the launcher execs; the package hint is per distro family. +REQUIRED_BINARIES = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session", + "xauth", "xdpyinfo", "setxkbmap") + +PACKAGES = { + "apt": ["tigervnc-standalone-server", "xfce4-panel", "xfwm4", "xfdesktop4", "xfce4-settings", + "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "xauth", "fonts-dejavu-core"], + "dnf": ["tigervnc-server-minimal", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", + "xfce4-terminal", "dbus-x11", "xorg-x11-server-utils", "xorg-x11-utils", "xorg-x11-xauth", + "dejavu-sans-fonts"], + "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", + "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xauth", "xorg-setxkbmap", "ttf-dejavu"], +} + + +def state_dir() -> Path: + return get_hermes_home() / "bot-desktop" + + +def is_supported_host() -> bool: + return sys.platform.startswith("linux") + + +def missing_binaries() -> list[str]: + return [b for b in REQUIRED_BINARIES if shutil.which(b) is None] + + +def package_manager() -> Optional[str]: + for pm in ("apt-get", "dnf", "pacman"): + if shutil.which(pm): + return "apt" if pm == "apt-get" else pm + return None + + +def install_command() -> Optional[str]: + pm = package_manager() + if pm is None: + return None + pkgs = " ".join(PACKAGES[pm]) + return { + "apt": f"sudo apt-get install -y --no-install-recommends {pkgs}", + "dnf": f"sudo dnf install -y {pkgs}", + "pacman": f"sudo pacman -S --needed --noconfirm {pkgs}", + }[pm] + + +@dataclass +class DesktopStatus: + profile: str + supported: bool + installed: bool + missing: list[str] + running: bool + pid: Optional[int] + display: Optional[str] + socket: Optional[str] + geometry: str + install_command: Optional[str] + + def as_dict(self) -> Dict[str, object]: + return dict(self.__dict__) + + +def _read(path: Path) -> Optional[str]: + try: + return path.read_text(encoding="utf-8").strip() or None + except OSError: + return None + + +def _pid_alive(pid: int) -> bool: + import psutil + return psutil.pid_exists(pid) + + +def _launcher_pid() -> Optional[int]: + raw = _read(state_dir() / "launcher.pid") + if not raw or not raw.isdigit(): + return None + pid = int(raw) + return pid if _pid_alive(pid) else None + + +def _display_in_use(num: int) -> bool: + return Path(f"/tmp/.X{num}-lock").exists() or Path(f"/tmp/.X11-unix/X{num}").exists() + + +def _allocate_display() -> int: + recorded = _read(state_dir() / "display") + if recorded and recorded.isdigit(): + return int(recorded) + for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1): + if not _display_in_use(num): + return num + raise RuntimeError("no free X display number in the Bot Desktop band") + + +def desktop_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: + """``base_env`` (default ``os.environ``) with this profile's DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS + merged in when its desktop is running. Unchanged otherwise, so hosts with a real seat keep it. + Pure: never starts anything (it is called from env builders, status probes and tests).""" + env = dict(os.environ if base_env is None else base_env) + published = published_env() + if published: + env.update(published) + env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends + return env + + +def ensure_started_for_tool() -> None: + """Tool-boundary hook (``computer_use`` dispatch): with ``bot_desktop.auto_start`` (default on) a Linux + host that has NO display and the packages installed gets its screen started on first use, so a headless + gateway works the first time instead of answering "no DISPLAY is set". Failure is not an error here; + the tool's own "no display" diagnosis is the right message then.""" + if published_env() or not _should_auto_start(os.environ): + return + try: + start() + except Exception as exc: + logger.info("Bot Desktop auto-start skipped: %s", exc) + + +def _should_auto_start(env: Dict[str, str]) -> bool: + if not is_supported_host() or env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"): + return False + if missing_binaries(): + return False + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly().get("bot_desktop") or {} + return bool(cfg.get("auto_start", True)) + + +def published_env() -> Dict[str, str]: + """Variables the launcher wrote once Xfce's private bus existed; empty when the desktop is down.""" + if _launcher_pid() is None: + return {} + raw = _read(state_dir() / "env") + if not raw: + return {} + out: Dict[str, str] = {} + for line in raw.splitlines(): + key, sep, value = line.partition("=") + if sep: + out[key.strip()] = value.strip() + return out + + +def rfb_socket_path() -> Optional[Path]: + sock = state_dir() / "rfb.sock" + return sock if _launcher_pid() is not None and sock.exists() else None + + +def geometry() -> str: + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly().get("bot_desktop") or {} + return str(cfg.get("geometry") or "1440x900") + + +def status(profile: Optional[str] = None) -> DesktopStatus: + missing: list[str] = missing_binaries() if is_supported_host() else list(REQUIRED_BINARIES) + pid = _launcher_pid() + env = published_env() + return DesktopStatus( + profile=profile or _profile_name(), + supported=is_supported_host(), + installed=not missing, + missing=missing, + running=pid is not None and bool(env.get("DISPLAY")), + pid=pid, + display=env.get("DISPLAY"), + socket=str(rfb_socket_path()) if rfb_socket_path() else None, + geometry=geometry(), + install_command=install_command() if missing else None, + ) + + +def _profile_name() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return get_active_profile_name() or "default" + except Exception: + return "default" + + +def start(*, wait_seconds: float = 15.0) -> DesktopStatus: + """Start this profile's desktop (idempotent). Blocks until the launcher publishes its env file or + ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker.""" + if not is_supported_host(): + raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") + missing = missing_binaries() + if missing: + hint = install_command() or "install TigerVNC (Xvnc) and the Xfce core components" + raise RuntimeError(f"Bot Desktop needs {', '.join(missing)} on the gateway host. Install: {hint}") + if _launcher_pid() is not None and published_env().get("DISPLAY"): + return status() + + sd = state_dir() + sd.mkdir(parents=True, exist_ok=True) + os.chmod(sd, 0o700) + num = _allocate_display() + (sd / "display").write_text(str(num), encoding="utf-8") + env_file = sd / "env" + env_file.unlink(missing_ok=True) + + child_env = {k: v for k, v in os.environ.items() if k not in { + "DISPLAY", "XAUTHORITY", "WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER"}} + child_env.update({ + "HERMES_BD_PROFILE": _profile_name(), + "HERMES_BD_DISPLAY_NUM": str(num), + "HERMES_BD_SOCKET": str(sd / "rfb.sock"), + "HERMES_BD_XAUTH": str(sd / "Xauthority"), + "HERMES_BD_ENV_FILE": str(env_file), + "HERMES_BD_CONFIG_HOME": str(sd / "xdg"), + "HERMES_BD_GEOMETRY": geometry(), + }) + log = open(sd / "launcher.log", "ab") # noqa: SIM115 — handed to the child, closed by it + proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host) + ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, + start_new_session=True, close_fds=True) + log.close() + (sd / "launcher.pid").write_text(str(proc.pid), encoding="utf-8") + + deadline = time.monotonic() + wait_seconds + while time.monotonic() < deadline: + if proc.poll() is not None: + tail = (sd / "launcher.log").read_bytes()[-2000:].decode("utf-8", "replace") + raise RuntimeError(f"Bot Desktop launcher exited with {proc.returncode}:\n{tail}") + if env_file.exists() and (sd / "rfb.sock").exists(): + logger.info("Bot Desktop for profile %s up on :%s", _profile_name(), num) + return status() + time.sleep(0.1) + raise RuntimeError(f"Bot Desktop did not publish its display within {wait_seconds:.0f}s (see {sd / 'launcher.log'})") + + +def stop() -> bool: + """Stop this profile's desktop; True when a running launcher was signalled.""" + pid = _launcher_pid() + sd = state_dir() + if pid is None: + (sd / "env").unlink(missing_ok=True) + return False + # The launcher runs in its own session; killing the group takes Xvnc, dbus and Xfce with it. + try: + os.killpg(pid, signal.SIGTERM) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + except ProcessLookupError: + pass + for _ in range(50): + if not _pid_alive(pid): + break + time.sleep(0.1) + else: + try: + os.killpg(pid, signal.SIGKILL) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + except ProcessLookupError: + pass + (sd / "launcher.pid").unlink(missing_ok=True) + (sd / "env").unlink(missing_ok=True) + return True diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 104bc7694df4..5a8c67b3abcb 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -42,7 +42,9 @@ def _build_browser_env() -> dict: env = hermes_subprocess_env(inherit_credentials=False) env.update({k: os.environ[k] for k in _BROWSER_PASSTHROUGH_KEYS if k in os.environ}) - return env + # Headed Chromium opens on this profile's Bot Desktop when one is running (human can take it over). + from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env + return _bot_desktop_env(env) try: diff --git a/tools/browser_tool_real_profile.py b/tools/browser_tool_real_profile.py index 103bc7a64cc7..dfbe6f0e4a02 100644 --- a/tools/browser_tool_real_profile.py +++ b/tools/browser_tool_real_profile.py @@ -163,12 +163,13 @@ def _launch_real_profile_chrome(real_binary: str, copy_dir: str) -> Tuple[Option except OSError: pass chrome_argv = [real_binary, f"--user-data-dir={copy_dir}", *_REAL_PROFILE_CHROME_FLAGS] - _has_display = bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + browser_env = _bt._build_browser_env() # carries the Bot Desktop DISPLAY when one is running + _has_display = bool(browser_env.get("DISPLAY") or browser_env.get("WAYLAND_DISPLAY")) if not (_cloud._is_headed_mode() and (_has_display or not sys.platform.startswith("linux"))): chrome_argv.append("--headless=new") try: chrome_proc = subprocess.Popen(chrome_argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, start_new_session=True, env=_bt._build_browser_env()) + stdin=subprocess.DEVNULL, start_new_session=True, env=browser_env) except (subprocess.SubprocessError, OSError) as e: return None, f"{_RP}the launch failed: {e}" _bt._real_profile_chrome_procs.append(chrome_proc) diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index 9be4b19cb77e..812d75560927 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -106,6 +106,10 @@ def cua_driver_child_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, the child has a Wayland display). Used by every spawn site (MCP, status, doctor, install) so CLI and gateway runtimes share one policy.""" env = dict(os.environ if base_env is None else base_env) + # A running Bot Desktop for this profile owns the agent's screen: DISPLAY/XAUTHORITY/DBUS point there so + # cua-driver never acts on a seat the human is sitting at (#90374 class) and headless hosts get a display. + from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env + env = _bot_desktop_env(env) if _cua_telemetry_disabled(): env[_CUA_TELEMETRY_ENV_VAR] = "0" if sys.platform == "linux" and env.get("WAYLAND_DISPLAY") and bool(_computer_use_cfg().get("native_wayland", False)): diff --git a/tools/computer_use/handoff.py b/tools/computer_use/handoff.py new file mode 100644 index 000000000000..6d04f048de1f --- /dev/null +++ b/tools/computer_use/handoff.py @@ -0,0 +1,38 @@ +"""Human handoff actions for ``computer_use`` on a Bot Desktop: ``request_handoff`` asks the person to +take over the screen (login, 2FA, CAPTCHA, payment) and ``wait_for_human`` blocks until control is +back. Both are answered without touching cua-driver, so a human typing a credential is never +captured. + +The notification itself (Desktop pane badge, Telegram/Discord message) is emitted by the gateway's +lease listener; this module only records intent on the lease and waits. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict + +from tools.bot_desktop import lease as _lease + +HANDOFF_ACTIONS = frozenset({"request_handoff", "wait_for_human"}) +_DEFAULT_WAIT_SECONDS = 600.0 +_MAX_WAIT_SECONDS = 1800.0 + + +def handle_handoff(action: str, args: Dict[str, Any]) -> str: + if action == "request_handoff": + reason = str(args.get("reason") or "The agent needs you to complete a step on its screen.").strip() + _lease.request_handoff(reason) + return json.dumps({ + "ok": True, "action": action, "state": _lease.get().as_dict(), + "next": "The user has been asked to open this bot's Desktop pane and take over. Call " + "computer_use action='wait_for_human' to block until they hand control back, then " + "re-capture before continuing — the screen state is whatever they left."}) + timeout = min(_MAX_WAIT_SECONDS, max(1.0, float(args.get("seconds") or _DEFAULT_WAIT_SECONDS))) + released = _lease.wait_for_release(timeout=timeout) + state = _lease.get().as_dict() + if released: + return json.dumps({"ok": True, "action": action, "state": state, + "next": "Control is back with you. Take a fresh capture; do not assume prior state."}) + return json.dumps({"ok": False, "action": action, "code": "still_waiting", "state": state, + "error": f"No hand-back within {timeout:.0f}s. Call wait_for_human again or ask the user in chat."}) diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index a38943c142bb..e984bfe3f60f 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -32,12 +32,16 @@ "list_apps", "list_windows", "focus_app", + "request_handoff", + "wait_for_human", ], "description": ( "Which action to perform. `capture` is free (no side effects). All other actions " "require approval unless auto-approved. Use `set_value` for select/popup elements and " "sliders — it selects the matching option directly without opening the native menu (no " - "focus steal)." + "focus steal). When a login, 2FA, CAPTCHA or payment step needs the human, call " + "`request_handoff` (with `reason`) so they can take over this screen from the Hermes " + "Desktop app, then `wait_for_human`; while they hold control every other action is refused." ), }, "mode": { @@ -141,13 +145,14 @@ ), }, "text": {"type": "string", "description": "Text to type (respects the current layout)."}, + "reason": {"type": "string", "description": "request_handoff: one sentence telling the human what to do on the screen (e.g. 'Sign in to LinkedIn and complete 2FA')."}, "keys": { "type": "string", "description": ( "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine." ), }, - "seconds": {"type": "number", "description": "Seconds to wait. Max 30."}, + "seconds": {"type": "number", "description": "wait: seconds to pause (max 30). wait_for_human: how long to block for the hand-back (default 600, max 1800)."}, "raise_window": { "type": "boolean", "description": ( diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index afb000913f17..e082c0b4c06c 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -248,6 +248,17 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: if not action: return json.dumps({"error": "missing `action`"}) session_id = str(kwargs.get("session_id") or "") # approval-state / daemon-mode isolation key + from tools.computer_use.handoff import HANDOFF_ACTIONS, handle_handoff + if action in HANDOFF_ACTIONS: + return handle_handoff(action, args) + # Bot Desktop lease: while a human drives the screen every action, capture included, is refused. + from tools.bot_desktop import lease as _bd_lease + from tools.bot_desktop.runtime import ensure_started_for_tool as _bd_ensure_started + try: + _bd_lease.assert_agent_may_act() + except _bd_lease.HumanHasControl as e: + return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)}) + _bd_ensure_started() # headless gateway: bring the profile's screen up before the backend probes DISPLAY if (err := _reject_unsafe(action, args)) is not None: return err scopes = ([action] if action in _ACTIONS and _ACTIONS[action].destructive else []) + ( diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py new file mode 100644 index 000000000000..9f9aede01000 --- /dev/null +++ b/tui_gateway/methods_display.py @@ -0,0 +1,121 @@ +"""Bot Desktop JSON-RPC handlers: the Desktop app's door to a profile's headless screen. + +``display.status`` reports runtime + lease; ``display.start`` / ``display.stop`` manage the Xvnc/Xfce +process; ``display.observe`` mints a single-use ticket the renderer redeems on ``/api/display/ws`` +(``hermes_cli.web_routers.display``) to stream raw RFB; ``display.lease.acquire`` / ``release`` are +Take over / Hand back. Every handler is profile-scoped so a multiplexed gateway answers for the bot +the pane is looking at. Lease transitions fan out as the global ``display.lease`` event so every +connected client repaints (badge on the bot row, red border on the viewer, agent handoff prompt). + +Bodies are rebound onto server.py's globals (method_ctx.bind_module) and reference them bare. +""" + +import logging +import threading + +from .method_ctx import HandlerRegistry, bind_module + +logger = logging.getLogger(__name__) +_registry = HandlerRegistry() +method = _registry.method +_profile_scoped = _registry.profile_scoped + +_DISPLAY_ERR = 5300 +_lease_listener_installed = threading.Event() + + +def _display_snapshot() -> dict: + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + st = _bd_runtime.status() + return {**st.as_dict(), "lease": _bd_lease.get().as_dict(), "profile_key": hermes_home_key()} + + +def _install_lease_listener() -> None: + """Once per process: broadcast every lease change to all connected clients.""" + if _lease_listener_installed.is_set(): + return + _lease_listener_installed.set() + from tools.bot_desktop import lease as _bd_lease + + def _on_change(profile_key: str, lease) -> None: + _broadcast_global_event("display.lease", {"profile_key": profile_key, "lease": lease.as_dict()}) + _bd_lease.on_change(_on_change) + + +@method("display.status") +@_profile_scoped +def _(rid, params: dict) -> dict: + _install_lease_listener() + try: + return _ok(rid, _display_snapshot()) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.start") +@_profile_scoped +def _(rid, params: dict) -> dict: + _install_lease_listener() + from tools.bot_desktop import runtime as _bd_runtime + try: + _bd_runtime.start() + return _ok(rid, _display_snapshot()) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.stop") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + try: + _bd_lease.release() + stopped = _bd_runtime.stop() + return _ok(rid, {**_display_snapshot(), "stopped": stopped}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.observe") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Mint a single-use, 30 s ticket for ``/api/display/ws``. The ticket carries the profile home so + the bridge dials THIS profile's RFB socket, and the viewer id so the lease can name the holder.""" + from hermes_constants import get_hermes_home + from hermes_cli.dashboard_auth.ws_tickets import mint_ticket + from tools.bot_desktop import runtime as _bd_runtime + try: + if _bd_runtime.rfb_socket_path() is None: + return _err(rid, _DISPLAY_ERR, "this profile's Bot Desktop is not running; call display.start first") + viewer_id = str(params.get("viewer_id") or "").strip() or f"viewer-{rid}" + ticket = mint_ticket(user_id=f"display:{viewer_id}", provider="bot-desktop", + extra={"hermes_home": str(get_hermes_home()), "viewer_id": viewer_id}) + return _ok(rid, {"ticket": ticket, "path": "/api/display/ws", "viewer_id": viewer_id, + **_display_snapshot()}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.lease.acquire") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease + viewer_id = str(params.get("viewer_id") or "").strip() + if not viewer_id: + return _err(rid, _DISPLAY_ERR, "viewer_id required") + lease = _bd_lease.acquire(viewer_id, reason=str(params.get("reason") or "")) + return _ok(rid, {"lease": lease.as_dict()}) + + +@method("display.lease.release") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease + viewer_id = str(params.get("viewer_id") or "").strip() or None + lease = _bd_lease.release(viewer_id) + return _ok(rid, {"lease": lease.as_dict()}) + + +def register(server) -> None: + bind_module(globals(), server, skip=("_",)) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index cbfe41572dbc..bb99b563fa76 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -3230,7 +3230,7 @@ def _resolve_name(name: str) -> str: methods_projects as _methods_projects, methods_session_foreign as _methods_session_foreign, methods_session_control as _methods_session_control, methods_subagents as _methods_subagents, methods_vault as _methods_vault, methods_free_tier as _methods_free_tier, - methods_connectors as _methods_connectors) + methods_connectors as _methods_connectors, methods_display as _methods_display) for _m in ( _session_transports, _session_reaper, _session_lifecycle, _session_workdir, _compute_host_bridge, _model_switch, @@ -3240,6 +3240,7 @@ def _resolve_name(name: str) -> str: _methods_browser_control, _methods_session, _methods_prompt, _methods_config, _methods_config_set, _methods_complete, _methods_tools, _methods_profiles, _methods_images, _methods_bot_relay, _prompt_turn, _billing_view, _methods_projects, _methods_session_foreign, - _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors): + _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors, + _methods_display): _m.register(sys.modules[__name__]) del _m diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md new file mode 100644 index 000000000000..6079927774cc --- /dev/null +++ b/website/docs/user-guide/features/bot-screen.md @@ -0,0 +1,121 @@ +--- +title: Bot Screen +sidebar_position: 17 +--- + +# Bot Screen + +On a headless Linux gateway host (a server, a cloud VM, Hermes Cloud) each bot +gets its **own desktop**: an Xfce screen the bot's `computer_use` and headed +browser act on, streamed live into Hermes Desktop. Watch what the bot does, +**take over** when it hits a login, 2FA prompt, CAPTCHA or payment step, then +**hand control back** and let it continue with the session you just signed in +to. The bot keeps working after you close the app or turn off your laptop; the +screen lives on the gateway host, not on your machine. + +Every Hermes profile ("bot") has its own screen, its own browser profile and +its own cookies. Screens are work surfaces, not security boundaries: the bots +share the host's user account, files and network (the same model as other +hosted-agent products). + +## Requirements + +- The gateway host runs Linux. macOS and Windows hosts already have a real + display; the pane is not offered there. +- TigerVNC's `Xvnc` and the Xfce core components are installed on the host. + Hermes Desktop and `hermes computer-use screen status` print the exact + package-manager line when they are missing; `hermes computer-use screen + install` runs it for you: + + | Distro | Packages | + |---|---| + | Debian / Ubuntu | `tigervnc-standalone-server xfce4-panel xfwm4 xfdesktop4 xfce4-settings xfce4-terminal dbus-x11 x11-xserver-utils x11-utils xauth fonts-dejavu-core` | + | Fedora | `tigervnc-server-minimal xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal dbus-x11 xorg-x11-server-utils xorg-x11-utils xorg-x11-xauth dejavu-sans-fonts` | + | Arch | `tigervnc xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal xorg-xsetroot xorg-xset xorg-xdpyinfo xorg-xauth xorg-setxkbmap ttf-dejavu` | + + Deliberately **not** the `xfce4` metapackage: it pulls in the screensaver, + power manager and polkit agent that lock or prompt a headless desktop. +- [Computer Use](./computer-use.md) enabled for the bot (cua-driver installed). + +## Using it + +1. In Hermes Desktop open **Bots**, right-click a bot, choose **Open Screen**. + The first time, click **Start screen** (or leave `bot_desktop.auto_start` on, + the default: the screen starts on the bot's first `computer_use` call when the + host has no display). A headed browser opens on the screen once it is running. +2. The pane streams the bot's desktop. The chip in the header says who is in + control: **Bot is in control** by default. +3. Click **Take over**. The border turns red, your keyboard and mouse now drive + the bot's screen. Sign in, solve the CAPTCHA, approve the payment. +4. Click **Hand back**. The bot regains control and re-captures the screen + before continuing. Closing the pane also hands control back. + +While you hold control the bot's `computer_use` calls (captures included) are +refused with `human_has_control`; the bot never sees what you type. + +The bot can ask for you: when it recognises a login or verification step it +calls `computer_use` with `action: "request_handoff"` and a reason, the pane +shows **Bot needs you**, and the bot blocks in `action: "wait_for_human"` until +you hand back. Your Telegram/Discord chat with the bot gets the same request. + +Two viewers on one screen: the most recent **Take over** wins; the previous +controller drops back to watching. + +## Browser sessions that survive the handoff + +The bot's headed Chromium (`browser.headed: true` or a real-profile session) +opens on the bot's screen and keeps one persistent profile per bot. What you +sign in to during a takeover is what the bot uses afterwards, and in every later +session for that bot, until the site itself expires the login. + +## CLI + +```bash +hermes computer-use screen status # installed? running? who holds control? +hermes computer-use screen start # start this profile's screen +hermes computer-use screen stop # stop it (hands control back first) +hermes computer-use screen install [-y] # apt/dnf/pacman the packages +hermes -p research computer-use screen start # another bot's screen +``` + +## Configuration + +```yaml +bot_desktop: + geometry: "1440x900" # screen size; the viewer scales to fit the pane + auto_start: true # start on the first computer_use call when the host has no display +``` + +State lives under `/bot-desktop/` per profile (RFB Unix socket, +Xauthority, launcher log, per-profile xfconf). + +## How it works + +- **TigerVNC `Xvnc`** is the X server and the RFB server in one process, per + profile, listening only on a `0600` Unix socket. No TCP port, no VNC + password: the gateway is the only process that can reach it. +- **Xfce** starts component-wise (`xfsettingsd`, `xfwm4 --compositor=off`, + `xfdesktop`, `xfce4-panel`) under a private D-Bus session, without + `xfce4-session`, so nothing tries to lock the screen or reach `logind`. +- **Hermes Desktop** bundles noVNC. It asks the gateway for a single-use ticket + (`display.observe`) over its normal authenticated connection and opens a + sibling WebSocket to `/api/display/ws`; the gateway splices the RFB stream + through. Nothing new is exposed; the pane works over local, SSH, URL+token + and Hermes Cloud connections alike. +- **Control lease.** The gateway drops keyboard, pointer and clipboard messages + from any viewer that does not hold the lease, at the RFB byte level; noVNC's + view-only flag is only the UI hint. The same lease gates `computer_use`. +- **Display binding.** The launcher publishes `DISPLAY`, `XAUTHORITY` and the + D-Bus address; every cua-driver and headed-browser spawn for that profile + inherits them, so the bot never acts on a display a human is sitting at. + +## Troubleshooting + +- **"Screen packages missing"** — run the printed install line on the gateway + host (not on the machine running Hermes Desktop). +- **Screen starts then stops** — read `/bot-desktop/launcher.log`. +- **Typing produces wrong characters** — the screen uses a US keymap so RFB + keysyms and cua-driver agree; change it with `setxkbmap` on that `DISPLAY` + if you need another layout. +- **Bot says `human_has_control` after you left** — click **Hand back** in the + pane, or `hermes computer-use screen stop` / `start`. diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md index 1148e2949a00..3bb53d2d2095 100644 --- a/website/docs/user-guide/features/computer-use.md +++ b/website/docs/user-guide/features/computer-use.md @@ -410,10 +410,11 @@ of screenshot context, not ~600K. [windows-ssh](https://cua.ai/docs/how-to-guides/driver/windows-ssh) has the recipe. - **Linux** requires a reachable display server. Headless servers - need Xvfb (`Xvfb :99 -screen 0 1920x1080x24`) before - `computer_use` can capture or inject events. Pure Wayland sessions - need an XWayland bridge for screen capture (cua-driver's Wayland - inject path handles input independently). + get one from [Bot Screen](./bot-screen.md): a per-profile Xfce + desktop over TigerVNC that Hermes starts on first use and streams + into Hermes Desktop, where you can take over for logins and 2FA. + Pure Wayland sessions need an XWayland bridge for screen capture + (cua-driver's Wayland inject path handles input independently). For cross-platform GUI automation without the desktop overhead (and without TCC / Session 0 / X11 setup), the `browser` toolset uses a diff --git a/website/sidebars.ts b/website/sidebars.ts index 55d1ed80160a..621205071ac5 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -121,6 +121,7 @@ const sidebars: SidebarsConfig = { 'user-guide/features/browser', 'user-guide/features/credential-vault', 'user-guide/features/computer-use', + 'user-guide/features/bot-screen', 'user-guide/features/vision', 'user-guide/features/image-generation', 'user-guide/features/spotify', From b3e8045813d13dcc720003e670294b8336054c85 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:24:30 -0700 Subject: [PATCH 02/55] fix(desktop): Bot Screen pane drops its RFB ref once noVNC closes itself After a server-side eviction (4000 control-taken) or stream loss noVNC has already torn the client down; detach() then called disconnect() on it and noVNC logged "Tried changing state of a disconnected RFB object". Clear the ref in the disconnect listener so teardown only touches a live client. --- apps/desktop/src/plugins/hermes-bots/screen-pane.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index 27d4190f9a99..d270cadd90f6 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -123,6 +123,12 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { } }) client.addEventListener('disconnect', event => { + // noVNC logs "Tried changing state of a disconnected RFB object" if we later call + // disconnect() on a client that already closed itself (eviction, stream loss). + if (rfb.current === client) { + rfb.current = null + } + if (generation !== attachGeneration.current) { return } From e89b00738a24986da994818cb0c9f97dacc6845d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:34:03 -0700 Subject: [PATCH 03/55] fix(desktop): declare @novnc/novnc types in the tracked vite-env.d.ts The declaration lived in src/types/novnc.d.ts, which .gitignore drops (apps/desktop/src/**/*.d.ts is ignored except for an allowlist), so the pushed tree failed typecheck with TS7016 while the local worktree passed. An ambient 'declare module' needs a script-scoped declaration file, so it joins vite-env.d.ts rather than the module-scoped global.d.ts. --- apps/desktop/src/vite-env.d.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/desktop/src/vite-env.d.ts b/apps/desktop/src/vite-env.d.ts index 11f02fe2a006..ad5fecb77017 100644 --- a/apps/desktop/src/vite-env.d.ts +++ b/apps/desktop/src/vite-env.d.ts @@ -1 +1,23 @@ /// + +// @novnc/novnc ships no typings (its export is core/rfb.js); the surface the Bot Screen pane uses. +// Declared here because `apps/desktop/src/**/*.d.ts` is gitignored except for the allowlisted files, +// and an ambient `declare module` only works in a script-scoped (import-free) declaration file. +declare module '@novnc/novnc' { + export default class RFB { + constructor(target: HTMLElement, urlOrChannel: string | WebSocket | RTCDataChannel, options?: Record) + viewOnly: boolean + scaleViewport: boolean + resizeSession: boolean + focusOnClick: boolean + background: string + qualityLevel: number + compressionLevel: number + addEventListener(type: string, listener: (event: CustomEvent) => void): void + removeEventListener(type: string, listener: (event: CustomEvent) => void): void + disconnect(): void + focus(): void + blur(): void + clipboardPasteFrom(text: string): void + } +} From 255f5c229323d84e1c9de24dcc68a096eae8612a Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 06:07:37 -0700 Subject: [PATCH 04/55] feat(bot-screen): Screen portal in routines + sessions, one-click package install from Desktop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways in, one install path, no shell required. - Screen portal box: a compact card ("Screen · Live / Stopped / Not installed on host", who holds control) rendered at the top of a bot's Scheduled Jobs pane above the routines, and under each gateway/profile group header in the Sessions sidebar (new `sidebar.gatewayGroup.header` contribution area, so the plugin owns the box and core only exposes the slot). Clicking it opens the Screen pane; it reads the same display.status/lease events as the pane. - Install from Desktop: `display.install` runs the distro package command on the gateway host (apt/dnf/pacman) as a supervised child, streams `display.install.log`, and finishes with `display.install.done`. When sudo needs a password the host raises `display.install.sudo.request` on the caller's own WebSocket; the renderer shows the existing masked SudoDialog (pointed at `display.install.sudo.respond`), so the password never touches the renderer store as plaintext beyond the field, is redacted from the gateway trace like `sudo.respond`, and an empty answer cancels without spawning the package manager. One install per profile at a time. The pane's "packages missing" state is now an install card with the command shown for the shell-inclined. - Opt-in stays intact: nothing installs on `hermes update`; the only triggers are the Desktop button and `hermes computer-use screen install`. Why the SudoDialog fallback to the app-level card: the install belongs to the connection, not to a chat turn, so the request has no session id; the focused chat's dialog now also shows the session-less card instead of it dying unseen. Live (headless Electron via CDP, `hermes serve` with the packages hidden): portal in Scheduled Jobs → click → Screen pane → Install on host → sudo card → Cancel → "Install cancelled" and the card returns; with the packages present: portal → Start → live stream, portal flips to "Live · bot in control"; the same portal renders under the `default` profile in the Sessions sidebar. --- .../src/app/chat/sidebar/gateway-groups.tsx | 35 ++++ apps/desktop/src/app/routes.ts | 19 ++ .../gateway-event/input-requests.ts | 14 +- .../src/components/prompt-overlays.tsx | 4 +- apps/desktop/src/i18n/ar.ts | 1 + apps/desktop/src/i18n/en.ts | 1 + apps/desktop/src/i18n/ja.ts | 1 + apps/desktop/src/i18n/ru.ts | 1 + apps/desktop/src/i18n/types.ts | 1 + apps/desktop/src/i18n/zh-hant.ts | 1 + apps/desktop/src/i18n/zh.ts | 1 + apps/desktop/src/plugins/hermes-bots/cron.tsx | 4 + apps/desktop/src/plugins/hermes-bots/i18n.ts | 73 +++++++- .../src/plugins/hermes-bots/plugin.tsx | 12 +- .../plugins/hermes-bots/screen-install.tsx | 110 +++++++++++ .../src/plugins/hermes-bots/screen-pane.tsx | 17 +- .../src/plugins/hermes-bots/screen-portal.tsx | 174 ++++++++++++++++++ apps/desktop/src/sdk/index.ts | 10 +- apps/desktop/src/store/prompts.ts | 9 +- tests/tools/test_bot_desktop_install.py | 47 +++++ tools/bot_desktop/install.py | 100 ++++++++++ tui_gateway/methods_display.py | 48 ++++- tui_gateway/methods_prompt.py | 2 +- tui_gateway/server.py | 2 +- .../docs/user-guide/features/bot-screen.md | 27 ++- 25 files changed, 676 insertions(+), 38 deletions(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-install.tsx create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-portal.tsx create mode 100644 tests/tools/test_bot_desktop_install.py create mode 100644 tools/bot_desktop/install.py diff --git a/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx b/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx index 82281dcd4126..65504ebb8399 100644 --- a/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx +++ b/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx @@ -5,6 +5,7 @@ import type { ReactNode } from 'react' import { useState } from 'react' import { type NewSessionSplitHandler, startNewSessionDrag } from '@/app/chat/new-session-drag' +import { type ProfileGroupHeaderContribution, SIDEBAR_PROFILE_GROUP_HEADER_AREA } from '@/app/routes' import { Button } from '@/components/ui/button' import { Codicon } from '@/components/ui/codicon' import { @@ -18,6 +19,8 @@ import { import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu' import { Input } from '@/components/ui/input' import { ProfileGlyph } from '@/components/ui/profile-glyph' +import { useContributions } from '@/contrib' +import { ContribBoundary, ContribRender } from '@/contrib/react/boundary' import type { SessionInfo } from '@/hermes' import { useI18n } from '@/i18n' import { useStoreSelector } from '@/lib/use-session-slice' @@ -272,6 +275,7 @@ function GatewayProfileGroup({ {open && ( <> {children} + {group.profile ? : null} {renderRows(sessions.slice(0, visibleCount))} {hiddenCount > 0 && ( ) } + +/** Plugin-contributed chrome at the top of one expanded gateway/profile group + * (`sidebar.profileGroup.header`): the Bots plugin mounts its Screen portal + * here so the profile's computer is one click away from its sessions. */ +function ProfileGroupHeaderSlot({ connectionId, profile }: { connectionId: null | string; profile: string }) { + const items = useContributions(SIDEBAR_PROFILE_GROUP_HEADER_AREA) + + if (!items.length) { + return null + } + + return ( +
+ {items.map(item => { + const data = item.data as Partial | undefined + + if (typeof data?.render !== 'function') { + return null + } + + const render = data.render + + return ( + + render({ connectionId, profile })} /> + + ) + })} +
+ ) +} diff --git a/apps/desktop/src/app/routes.ts b/apps/desktop/src/app/routes.ts index 7e5fbd471826..c3a08f529899 100644 --- a/apps/desktop/src/app/routes.ts +++ b/apps/desktop/src/app/routes.ts @@ -123,6 +123,25 @@ export interface SidebarNavContribution { path: string } +// ── Contributed profile-group header — the `sidebar.profileGroup.header` area ─ +// A RENDER contribution mounted at the top of each gateway/profile group in the +// Sessions sidebar (above its session rows) while the group is expanded. The +// contribution's `data` is a `ProfileGroupHeaderContribution`; core calls +// `render(route)` with the group's connection + profile so one contribution +// serves every group. First consumer: the Bots plugin's Screen portal. + +export const SIDEBAR_PROFILE_GROUP_HEADER_AREA = 'sidebar.profileGroup.header' + +export interface ProfileGroupRoute { + connectionId: null | string + profile: string +} + +/** Payload of a `sidebar.profileGroup.header` data contribution. */ +export interface ProfileGroupHeaderContribution { + render: (route: ProfileGroupRoute) => ReactNode +} + // Views that render as a full-screen modal card (OverlayView) over the shell. // While one is open the app's titlebar control clusters must hide so they don't // bleed over the overlay (they sit at a higher z-index than the overlay card). diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts index 9319993797ca..e6397a32b84f 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts @@ -313,13 +313,19 @@ export function handleInputRequestEvent(ctx: GatewayEventContext): boolean { return true } - if (event.type === 'sudo.request') { - // Sudo password capture (tools/terminal_tool.py). Blocked on - // sudo.respond {request_id, password}. + if (event.type === 'sudo.request' || event.type === 'display.install.sudo.request') { + // Sudo password capture (tools/terminal_tool.py), or the Bot Screen package install + // (tui_gateway/methods_display.py) reusing the same masked card. Blocked on + // .respond {request_id, password}. const requestId = typeof payload?.request_id === 'string' ? payload.request_id : '' + const install = event.type === 'display.install.sudo.request' if (requestId) { - setSudoRequest({ requestId, sessionId: sessionId ?? null }) + setSudoRequest({ + requestId, + sessionId: sessionId ?? null, + ...(install ? { respondMethod: 'display.install.sudo.respond', description: translateNow('prompts.sudoInstallDesc') } : {}) + }) if (sessionId) { updateSessionState(sessionId, state => ({ ...state, needsInput: true })) diff --git a/apps/desktop/src/components/prompt-overlays.tsx b/apps/desktop/src/components/prompt-overlays.tsx index 091e306ad5c7..0e76b01249f9 100644 --- a/apps/desktop/src/components/prompt-overlays.tsx +++ b/apps/desktop/src/components/prompt-overlays.tsx @@ -78,7 +78,7 @@ function SudoDialog({ sessionId }: { sessionId: string | null }) { setSubmitting(true) try { - await gateway.request<{ status?: string }>('sudo.respond', { + await gateway.request<{ status?: string }>(request.respondMethod ?? 'sudo.respond', { password: value, request_id: request.requestId }) @@ -126,7 +126,7 @@ function SudoDialog({ sessionId }: { sessionId: string | null }) { {copy.sudoTitle} - {copy.sudoDesc} + {request.description ?? copy.sudoDesc}
diff --git a/apps/desktop/src/i18n/ar.ts b/apps/desktop/src/i18n/ar.ts index 6d70110018d1..47c44c9ebb39 100644 --- a/apps/desktop/src/i18n/ar.ts +++ b/apps/desktop/src/i18n/ar.ts @@ -3046,6 +3046,7 @@ export const ar = defineLocale({ secretSendFailed: 'فشل إرسال السر', sudoTitle: 'مطلوب sudo', sudoDesc: 'أدخل كلمة المرور لمتابعة الأمر.', + sudoInstallDesc: 'يحتاج Hermes إلى كلمة مرور sudo لتثبيت حزم Bot Screen (TigerVNC + Xfce) على مضيف البوابة. تُرسل إلى ذلك المضيف فقط.', sudoPlaceholder: 'كلمة المرور', secretTitle: 'مطلوب سر', secretDesc: 'أدخل القيمة المطلوبة لمتابعة المهمة.', diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index acbde2d1142b..a5500c5e4775 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -4011,6 +4011,7 @@ export const en: Translations = { secretSendFailed: 'Could not send secret', sudoTitle: 'Administrator password', sudoDesc: 'Hermes needs your sudo password to run a privileged command. It is sent only to your local agent.', + sudoInstallDesc: 'Hermes needs your sudo password to install the Bot Screen packages (TigerVNC + Xfce) on the gateway host. It is sent only to that host.', sudoPlaceholder: 'sudo password', secretTitle: 'Secret required', secretDesc: 'Hermes needs a credential to continue.', diff --git a/apps/desktop/src/i18n/ja.ts b/apps/desktop/src/i18n/ja.ts index cdd512551c88..e870777c436a 100644 --- a/apps/desktop/src/i18n/ja.ts +++ b/apps/desktop/src/i18n/ja.ts @@ -3456,6 +3456,7 @@ export const ja = defineLocale({ sudoTitle: '管理者パスワード', sudoDesc: 'Hermes は特権コマンドを実行するために sudo パスワードが必要です。ローカルエージェントにのみ送信されます。', + sudoInstallDesc: 'Bot Screen のパッケージ（TigerVNC + Xfce）をゲートウェイホストにインストールするため、sudo パスワードが必要です。そのホストにのみ送信されます。', sudoPlaceholder: 'sudo パスワード', secretTitle: 'シークレットが必要です', secretDesc: 'Hermes は続行するための認証情報が必要です。', diff --git a/apps/desktop/src/i18n/ru.ts b/apps/desktop/src/i18n/ru.ts index e8ffa262a36c..5f7a04d42b25 100644 --- a/apps/desktop/src/i18n/ru.ts +++ b/apps/desktop/src/i18n/ru.ts @@ -3753,6 +3753,7 @@ export const ru = defineLocale({ sudoTitle: 'Пароль администратора', sudoDesc: 'Hermes нужен ваш пароль sudo, чтобы выполнить команду с повышенными правами. Он отправляется только вашему локальному агенту.', + sudoInstallDesc: 'Hermes нужен ваш пароль sudo, чтобы установить пакеты Bot Screen (TigerVNC + Xfce) на хосте шлюза. Он отправляется только на этот хост.', sudoPlaceholder: 'пароль sudo', secretTitle: 'Требуется секрет', secretDesc: 'Hermes нужны учётные данные, чтобы продолжить.', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 0c16cb4b069a..e8b4a3eb25b7 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -3505,6 +3505,7 @@ export interface Translations { secretSendFailed: string sudoTitle: string sudoDesc: string + sudoInstallDesc: string sudoPlaceholder: string secretTitle: string secretDesc: string diff --git a/apps/desktop/src/i18n/zh-hant.ts b/apps/desktop/src/i18n/zh-hant.ts index 4759253d154e..21bc27f7fa61 100644 --- a/apps/desktop/src/i18n/zh-hant.ts +++ b/apps/desktop/src/i18n/zh-hant.ts @@ -3311,6 +3311,7 @@ export const zhHant = defineLocale({ secretSendFailed: '無法傳送密鑰', sudoTitle: '管理員密碼', sudoDesc: 'Hermes 需要您的 sudo 密碼來執行特權指令。它只會傳送給您的本機代理。', + sudoInstallDesc: 'Hermes 需要您的 sudo 密碼，以在閘道主機上安裝 Bot Screen 套件（TigerVNC + Xfce）。它只會傳送到該主機。', sudoPlaceholder: 'sudo 密碼', secretTitle: '需要密鑰', secretDesc: 'Hermes 需要一個憑證才能繼續。', diff --git a/apps/desktop/src/i18n/zh.ts b/apps/desktop/src/i18n/zh.ts index 7e78217d1b7c..26f00c698daa 100644 --- a/apps/desktop/src/i18n/zh.ts +++ b/apps/desktop/src/i18n/zh.ts @@ -4124,6 +4124,7 @@ export const zh = defineLocale({ secretSendFailed: '无法发送密钥', sudoTitle: '管理员密码', sudoDesc: 'Hermes 需要你的 sudo 密码来运行特权命令。它只会发送给你的本地 agent。', + sudoInstallDesc: 'Hermes 需要你的 sudo 密码，以在网关主机上安装 Bot Screen 软件包（TigerVNC + Xfce）。它只会发送到该主机。', sudoPlaceholder: 'sudo 密码', secretTitle: '需要密钥', secretDesc: 'Hermes 需要一个凭据才能继续。', diff --git a/apps/desktop/src/plugins/hermes-bots/cron.tsx b/apps/desktop/src/plugins/hermes-bots/cron.tsx index 931bd069ac60..74ec2deb2aee 100644 --- a/apps/desktop/src/plugins/hermes-bots/cron.tsx +++ b/apps/desktop/src/plugins/hermes-bots/cron.tsx @@ -45,6 +45,7 @@ import { labeled } from './dialog-parts' import { botsText, useBots } from './i18n' import { displayName } from './labels' import { botConnectionRoute, botRosterMeta, requestForBot } from './routing' +import { ScreenPortal } from './screen-portal' import { ID } from './shared' import type { BotMeta, RosterRow, RoutineJob } from './types' @@ -1265,6 +1266,9 @@ export function RoutinesPane() {
+
+ +
{staleNotice ? (
{staleNotice} diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index 12e63a6b71fc..43550dd1ddc5 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -232,6 +232,19 @@ type BotsMessages = { notInstalledTitle: string notInstalledBody: string installHint: string + install: string + installing: string + installCancelled: string + installFailed: string + noPackageManager: string + portalTitle: string + portalOpen: string + portalWatching: string + portalYouControl: string + portalOtherControls: string + portalStopped: string + portalNotInstalled: string + portalUnsupported: string recheck: string stoppedTitle: string stoppedBody: string @@ -478,7 +491,20 @@ const en: BotsMessages = { unsupportedBody: 'Bot screens run on Linux gateway hosts. This bot uses the host\u2019s own display.', notInstalledTitle: 'Screen packages missing', notInstalledBody: 'The gateway host needs TigerVNC and the Xfce core to give this bot a screen. Run on the host:', - installHint: 'or: hermes computer-use screen install', + installHint: 'Runs on the gateway host as the user Hermes runs as; sudo is asked for once, through Hermes.', + install: 'Install on host', + installing: 'Installing…', + installCancelled: 'Install cancelled: no sudo password was provided.', + installFailed: 'Install failed. Read the log above, or run the command on the host yourself.', + noPackageManager: 'No supported package manager (apt, dnf, pacman) was found on the gateway host.', + portalTitle: 'Screen', + portalOpen: 'Open', + portalWatching: 'Live · bot in control', + portalYouControl: 'Live · you are in control', + portalOtherControls: 'Live · another viewer in control', + portalStopped: 'Stopped', + portalNotInstalled: 'Not installed on host', + portalUnsupported: 'Not available on this host', recheck: 'Check again', stoppedTitle: 'Screen is off', stoppedBody: 'Start this bot\u2019s desktop to watch what it does and take over when it needs you.', @@ -720,7 +746,20 @@ const ja: BotsMessages = { unsupportedBody: 'ボット画面は Linux のゲートウェイホストで動作します。このボットはホスト自身のディスプレイを使います。', notInstalledTitle: '画面パッケージが不足しています', notInstalledBody: 'このボットに画面を与えるには、ゲートウェイホストに TigerVNC と Xfce コアが必要です。ホストで実行:', - installHint: 'または: hermes computer-use screen install', + installHint: 'Hermes を実行しているユーザーとしてゲートウェイホスト上で実行されます。sudo は Hermes 経由で一度だけ求められます。', + install: 'ホストにインストール', + installing: 'インストール中…', + installCancelled: 'インストールを中止しました: sudo パスワードが入力されませんでした。', + installFailed: 'インストールに失敗しました。上のログを確認するか、ホストでコマンドを直接実行してください。', + noPackageManager: 'ゲートウェイホストに対応するパッケージマネージャー (apt, dnf, pacman) が見つかりません。', + portalTitle: 'スクリーン', + portalOpen: '開く', + portalWatching: 'ライブ · ボットが操作中', + portalYouControl: 'ライブ · あなたが操作中', + portalOtherControls: 'ライブ · 別のビューアーが操作中', + portalStopped: '停止中', + portalNotInstalled: 'ホストに未インストール', + portalUnsupported: 'このホストでは利用できません', recheck: '再確認', stoppedTitle: '画面はオフです', stoppedBody: 'このボットのデスクトップを起動すると、動作を見守り、必要なときに操作を引き継げます。', @@ -957,7 +996,20 @@ const zh: BotsMessages = { unsupportedBody: '机器人屏幕在 Linux 网关主机上运行。此机器人使用主机自身的显示器。', notInstalledTitle: '缺少屏幕软件包', notInstalledBody: '网关主机需要 TigerVNC 和 Xfce 核心组件才能为此机器人提供屏幕。在主机上运行:', - installHint: '或: hermes computer-use screen install', + installHint: '在网关主机上以运行 Hermes 的用户身份执行；sudo 只会通过 Hermes 询问一次。', + install: '安装到主机', + installing: '正在安装…', + installCancelled: '安装已取消：未提供 sudo 密码。', + installFailed: '安装失败。请查看上方日志，或在主机上手动运行该命令。', + noPackageManager: '网关主机上未找到受支持的包管理器（apt、dnf、pacman）。', + portalTitle: '屏幕', + portalOpen: '打开', + portalWatching: '直播 · 机器人控制中', + portalYouControl: '直播 · 你在控制', + portalOtherControls: '直播 · 其他查看者控制中', + portalStopped: '已停止', + portalNotInstalled: '主机未安装', + portalUnsupported: '此主机不可用', recheck: '重新检查', stoppedTitle: '屏幕已关闭', stoppedBody: '启动此机器人的桌面，观看它的操作，并在需要时接管。', @@ -1194,7 +1246,20 @@ const zhHant: BotsMessages = { unsupportedBody: '機器人螢幕在 Linux 閘道主機上執行。此機器人使用主機自身的顯示器。', notInstalledTitle: '缺少螢幕套件', notInstalledBody: '閘道主機需要 TigerVNC 與 Xfce 核心元件才能為此機器人提供螢幕。在主機上執行:', - installHint: '或: hermes computer-use screen install', + installHint: '在閘道主機上以執行 Hermes 的使用者身分執行；sudo 只會透過 Hermes 詢問一次。', + install: '安裝到主機', + installing: '安裝中…', + installCancelled: '安裝已取消：未提供 sudo 密碼。', + installFailed: '安裝失敗。請查看上方日誌，或在主機上手動執行該指令。', + noPackageManager: '閘道主機上找不到受支援的套件管理器（apt、dnf、pacman）。', + portalTitle: '螢幕', + portalOpen: '開啟', + portalWatching: '直播 · 機器人控制中', + portalYouControl: '直播 · 您在控制', + portalOtherControls: '直播 · 其他檢視者控制中', + portalStopped: '已停止', + portalNotInstalled: '主機未安裝', + portalUnsupported: '此主機不可用', recheck: '重新檢查', stoppedTitle: '螢幕已關閉', stoppedBody: '啟動此機器人的桌面，觀看它的操作，並在需要時接手。', diff --git a/apps/desktop/src/plugins/hermes-bots/plugin.tsx b/apps/desktop/src/plugins/hermes-bots/plugin.tsx index 8032e16134ff..d472473b6dff 100644 --- a/apps/desktop/src/plugins/hermes-bots/plugin.tsx +++ b/apps/desktop/src/plugins/hermes-bots/plugin.tsx @@ -15,8 +15,8 @@ * bot-initiated sends use `hermes -p chat --in ~ -c "Bot Chat"`. */ -import { CHAT_EMPTY_AREA, COMPOSER_AREAS, host, PALETTE_AREA, translateNow } from '@hermes/plugin-sdk' -import type { ChatEmptyProps, PluginContext } from '@hermes/plugin-sdk' +import { CHAT_EMPTY_AREA, COMPOSER_AREAS, host, PALETTE_AREA, SIDEBAR_PROFILE_GROUP_HEADER_AREA, translateNow } from '@hermes/plugin-sdk' +import type { ChatEmptyProps, PluginContext, ProfileGroupRoute } from '@hermes/plugin-sdk' import { startFaceClock, stopFaceClock } from './avatar' import { @@ -69,6 +69,7 @@ import { sessionOwnsWorkspace } from './roster-pane' import { botRosterMeta, botWorkspaceOwnerKey, setBotsWorkspaceOwner } from './routing' +import { ProfileGroupScreenPortal } from './screen-portal' import { startHideSweepScheduler } from './session-sweep' import { bumpBotOpenGeneration, getBotOpenGeneration, ID, setPluginCtx } from './shared' import type { GroupChat, RosterRow } from './types' @@ -363,6 +364,13 @@ export default { // the meta/room storage hydrates above have landed; idempotent after that. // (Feature-guarded: bare vm test harnesses have no setTimeout global.) startHideSweepScheduler(ctx) + // Sessions sidebar: each gateway/profile group gets the profile's Screen portal + // above its sessions, so the bot's computer is reachable from either mode. + ctx.register({ + id: 'screen-portal', + area: SIDEBAR_PROFILE_GROUP_HEADER_AREA, + data: { render: (route: ProfileGroupRoute) => } + }) ctx.register({ id: 'pane', area: 'panes', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx new file mode 100644 index 000000000000..11662c0df86c --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx @@ -0,0 +1,110 @@ +/** + * Bot Screen install card — installs the TigerVNC + Xfce packages on the bot's + * gateway host from inside Hermes Desktop. + * + * `display.install` starts the distro package command on the host; sudo, when + * needed, arrives as the same masked password card the terminal tool uses + * (`display.install.sudo.request`), so the password never touches this pane. + * Output streams back as `display.install.log`; `display.install.done` carries + * a fresh status the caller uses to flip the pane to "Start screen". + */ + +import { Button, Codicon, GlyphSpinner, host } from '@hermes/plugin-sdk' +import type { RpcEvent } from '@hermes/plugin-sdk' +import { useCallback, useEffect, useRef, useState } from 'react' + +import { useBots } from './i18n' +import { displayRequest, type DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +const LOG_KEEP = 200 + +interface ScreenInstallCardProps { + bot: RosterRow + status: DisplayStatus + onInstalled: (status: DisplayStatus) => void +} + +export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCardProps) { + const t = useBots() + const [phase, setPhase] = useState<'idle' | 'running' | 'failed'>('idle') + const [log, setLog] = useState([]) + const [error, setError] = useState(null) + const logEnd = useRef(null) + + useEffect(() => { + logEnd.current?.scrollIntoView({ block: 'end' }) + }, [log]) + + useEffect(() => { + const offLog = host.onEvent('display.install.log', (event: RpcEvent) => { + const payload = event.payload as { profile_key?: string; line?: string } | undefined + + if (payload?.profile_key === status.profile_key && typeof payload.line === 'string') { + const line = payload.line + setLog(prev => (prev.length >= LOG_KEEP ? [...prev.slice(1), line] : [...prev, line])) + } + }) + const offDone = host.onEvent('display.install.done', (event: RpcEvent) => { + const payload = event.payload as { profile_key?: string; code?: number; status?: DisplayStatus } | undefined + + if (payload?.profile_key !== status.profile_key) { + return + } + + if (payload.code === 0 && payload.status?.installed) { + setPhase('idle') + onInstalled(payload.status) + } else { + setPhase('failed') + setError(payload.code === -1 ? t.screen.installCancelled : t.screen.installFailed) + } + }) + + return () => { + offLog() + offDone() + } + }, [onInstalled, status.profile_key, t.screen.installCancelled, t.screen.installFailed]) + + const install = useCallback(async () => { + setPhase('running') + setLog([]) + setError(null) + + try { + await displayRequest(bot, 'display.install') + } catch (err) { + setPhase('failed') + setError(err instanceof Error ? err.message : String(err)) + } + }, [bot]) + + return ( +
+
+
{t.screen.notInstalledTitle}
+
{t.screen.notInstalledBody}
+ {status.install_command ? ( + {status.install_command} + ) : ( +
{t.screen.noPackageManager}
+ )} + {status.install_command ? ( + + ) : null} + {log.length > 0 ? ( + 
+            {log.join('\n')}
+            

+
+ ) : null} + {error ?
{error}
: null} +
{t.screen.installHint}
+
+
+ ) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index d270cadd90f6..e4cf6a12e9b9 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -16,6 +16,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useBots } from './i18n' import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, resolveScreenWsUrl, VIEWER_ID } from './screen-connection' +import { ScreenInstallCard } from './screen-install' import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' import type { RosterRow } from './types' @@ -224,21 +225,7 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { } if (status && !status.installed) { - return ( -
-
-
{t.screen.notInstalledTitle}
-
{t.screen.notInstalledBody}
- {status.install_command ? ( - {status.install_command} - ) : null} -
{t.screen.installHint}
- -
-
- ) + return setScreenStatus(bot, next)} status={status} /> } if (status && !status.running) { diff --git a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx new file mode 100644 index 000000000000..3204de79064a --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx @@ -0,0 +1,174 @@ +/** + * Screen portal — the compact "this bot's computer" box that sits above a + * bot's routines and on a gateway/profile group in the Sessions sidebar. + * One click opens the live Screen pane; the subtitle says whether the screen + * is live and who holds it, so the user knows before opening whether they + * are about to watch, take over, install or start. + * + * Reads the same per-bot cache the pane paints (`$screenState`) and refreshes + * it once on mount so a bot the user never opened still shows real state. + */ + +import { Codicon, host, useValue } from '@hermes/plugin-sdk' +import type { RpcEvent } from '@hermes/plugin-sdk' +import type { ProfileGroupRoute } from '@hermes/plugin-sdk' +import { useEffect } from 'react' + +import { $lastRoster } from './data' +import { useBots } from './i18n' +import { resolveBotConnectionRoute } from './routing' +import { type DisplayLease, displayRequest, type DisplayStatus, VIEWER_ID } from './screen-connection' +import { openBotScreen } from './screen-open' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' +import type { BotMeta, RosterRow } from './types' + +export type PortalTone = 'live' | 'human' | 'other' | 'off' | 'missing' | 'unsupported' | 'unknown' + +/** Pure: map cached status + lease to what the portal says. */ +export function portalTone(status: DisplayStatus | null, lease: DisplayLease | null): PortalTone { + if (!status) { + return 'unknown' + } + + if (!status.supported) { + return 'unsupported' + } + + if (!status.installed) { + return 'missing' + } + + if (!status.running) { + return 'off' + } + + if (lease?.holder === 'human') { + return lease.viewer_id === VIEWER_ID ? 'human' : 'other' + } + + return 'live' +} + +const TONE_ICON: Record = { + live: 'device-desktop', + human: 'record-keys', + other: 'eye', + off: 'debug-stop', + missing: 'cloud-download', + unsupported: 'circle-slash', + unknown: 'device-desktop' +} + +const TONE_DOT: Record = { + live: 'bg-emerald-500', + human: 'bg-red-500', + other: 'bg-amber-500', + off: 'bg-(--ui-text-quaternary)', + missing: 'bg-(--ui-text-quaternary)', + unsupported: 'bg-(--ui-text-quaternary)', + unknown: 'bg-(--ui-text-quaternary)' +} + +export function useScreenPortalState(bot: RosterRow) { + const all = useValue($screenState) + const state = screenStateFor(all, bot) + const status = state?.status ?? null + const profileKey = status?.profile_key + + useEffect(() => { + if (status) { + return + } + + let cancelled = false + + void displayRequest(bot, 'display.status') + .then(next => { + if (!cancelled) { + setScreenStatus(bot, next) + } + }) + .catch(() => { + /* offline bot / older backend: the portal stays in its unknown state */ + }) + + return () => { + cancelled = true + } + }, [bot, status]) + + useEffect( + () => + host.onEvent('display.lease', (event: RpcEvent) => { + const payload = event.payload as { profile_key?: string; lease?: DisplayLease } | undefined + + if (payload?.lease && payload.profile_key && payload.profile_key === profileKey) { + setScreenLease(bot, payload.lease) + } + }), + [bot, profileKey] + ) + + return { status, lease: state?.lease ?? null, tone: portalTone(status, state?.lease ?? null) } +} + +export function ScreenPortal({ bot, meta, compact = false }: { bot: RosterRow; meta?: BotMeta | null; compact?: boolean }) { + const t = useBots() + const { status, tone } = useScreenPortalState(bot) + + const subtitle = { + live: t.screen.portalWatching, + human: t.screen.portalYouControl, + other: t.screen.portalOtherControls, + off: t.screen.portalStopped, + missing: t.screen.portalNotInstalled, + unsupported: t.screen.portalUnsupported, + unknown: status?.display ?? '' + }[tone] + + if (tone === 'unsupported' && compact) { + return null + } + + return ( + + ) +} + +/** Sessions-sidebar variant: the gateway/profile group hands us its route; find the + * bot that owns it, or synthesize a scoped row so a profile outside the current + * roster filter still gets its portal (the portal only needs a routable row). */ +export function ProfileGroupScreenPortal({ route }: { route: ProfileGroupRoute }) { + const roster = useValue($lastRoster) + + const bot = + roster.find(row => { + const resolved = resolveBotConnectionRoute(row) + + return resolved.route + ? resolved.route.profile === route.profile && (route.connectionId === null || resolved.route.connectionId === route.connectionId) + : row.name === route.profile && route.connectionId === null + }) ?? + (route.connectionId + ? ({ name: route.profile, sourceScoped: true, connectionId: route.connectionId, connectionKind: route.connectionId === 'local' ? 'local' : 'remote' } as RosterRow) + : ({ name: route.profile } as RosterRow)) + + return +} diff --git a/apps/desktop/src/sdk/index.ts b/apps/desktop/src/sdk/index.ts index b3b2f85abf01..39ca078d0e21 100644 --- a/apps/desktop/src/sdk/index.ts +++ b/apps/desktop/src/sdk/index.ts @@ -1501,7 +1501,15 @@ export { PanelRowMenu, PanelSectionLabel } from '@/app/overlays/panel' -export { type RouteContribution, ROUTES_AREA, SIDEBAR_NAV_AREA, type SidebarNavContribution } from '@/app/routes' +export { + type ProfileGroupHeaderContribution, + type ProfileGroupRoute, + type RouteContribution, + ROUTES_AREA, + SIDEBAR_NAV_AREA, + SIDEBAR_PROFILE_GROUP_HEADER_AREA, + type SidebarNavContribution +} from '@/app/routes' /** THE full per-toolset config panel core Settings renders — provider picker, * env vars / API keys, model catalog picker, and post-setup runners. Route- diff --git a/apps/desktop/src/store/prompts.ts b/apps/desktop/src/store/prompts.ts index 7325c4f54ed8..e57fbf660bbd 100644 --- a/apps/desktop/src/store/prompts.ts +++ b/apps/desktop/src/store/prompts.ts @@ -98,6 +98,11 @@ interface PendingApprovalPayload { export interface SudoRequest extends KeyedPrompt { requestId: string + /** JSON-RPC method that resolves this request; the terminal tool's `sudo.respond` by default, + * `display.install.sudo.respond` for a Bot Screen package install. */ + respondMethod?: string + /** Description override so the card can say WHAT the password is for. */ + description?: string } export interface SecretRequest extends KeyedPrompt { @@ -225,8 +230,10 @@ export async function replayPendingApproval(gateway: ApprovalGateway | null, ses * active-session `$*Request` views (same map, fixed key). */ export const sessionApprovalRequest = (sessionId: string | null) => computed(approval.$all, all => all[keyFor(sessionId)] ?? null) +/** A session's sudo card, else the app-level one (a Bot Screen package install is raised with no + * session: it belongs to the connection, not to a turn, so whichever chat is focused shows it). */ export const sessionSudoRequest = (sessionId: string | null) => - computed(sudo.$all, all => all[keyFor(sessionId)] ?? null) + computed(sudo.$all, all => all[keyFor(sessionId)] ?? (sessionId ? all[keyFor(null)] ?? null : null)) export const sessionSecretRequest = (sessionId: string | null) => computed(secret.$all, all => all[keyFor(sessionId)] ?? null) diff --git a/tests/tools/test_bot_desktop_install.py b/tests/tools/test_bot_desktop_install.py new file mode 100644 index 000000000000..5b0912d7e827 --- /dev/null +++ b/tests/tools/test_bot_desktop_install.py @@ -0,0 +1,47 @@ +"""Bot Desktop package install: sudo hand-off and single-flight invariants.""" + +from __future__ import annotations + +import threading + +import pytest + +from tools.bot_desktop import install, runtime + + +@pytest.fixture(autouse=True) +def _isolated_host(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(install, "_sudo_nopasswd", lambda: False) + monkeypatch.setattr(runtime, "is_supported_host", lambda: True) + monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y tigervnc-standalone-server") + yield + install._running.clear() + + +def test_empty_password_cancels_without_spawning(monkeypatch): + monkeypatch.setattr(install.subprocess, "Popen", lambda *a, **k: pytest.fail("package manager spawned")) + lines: list[str] = [] + code = install.install_packages(ask_password=lambda: "", on_line=lines.append) + assert code == -1 + assert any("cancelled" in line for line in lines) + + +def test_second_install_for_same_profile_is_refused(monkeypatch): + gate = threading.Event() + entered = threading.Event() + + def slow_run(cmd, *, ask_password, on_line, timeout_seconds): + entered.set() + gate.wait(5) + return 0 + + monkeypatch.setattr(install, "_run", slow_run) + worker = threading.Thread(target=install.install_packages, kwargs={"ask_password": lambda: "pw", "on_line": lambda _l: None}) + worker.start() + assert entered.wait(5) + with pytest.raises(install.InstallBusy): + install.install_packages(ask_password=lambda: "pw", on_line=lambda _l: None) + gate.set() + worker.join(5) + install.assert_not_running() # slot released once the run finishes diff --git a/tools/bot_desktop/install.py b/tools/bot_desktop/install.py new file mode 100644 index 000000000000..ef374a62e3ec --- /dev/null +++ b/tools/bot_desktop/install.py @@ -0,0 +1,100 @@ +"""Install the Bot Desktop packages on the gateway host from a Desktop client. + +The install runs the distro command from ``runtime.install_command()`` (apt/dnf/pacman) as a child +process on THIS host. Privilege comes from the same masked ``sudo.request`` card the terminal tool +raises: ``sudo -n true`` is probed first (NOPASSWD / cached timestamp hosts never see a prompt); when +a password is needed the caller-supplied ``ask_password`` blocks on the card and the value is written +to sudo's stdin (``-S``) exactly once, never logged, never placed on the command line. Output lines +stream through ``on_line`` so the pane can show apt's progress; the return value is the exit code. + +One install per profile at a time; a second request while one runs is refused. +""" + +from __future__ import annotations + +import logging +import os +import shlex +import subprocess +import threading +from typing import Callable, Optional + +from hermes_constants import hermes_home_key +from tools.bot_desktop import runtime + +logger = logging.getLogger(__name__) + +_install_lock = threading.Lock() +_running: set[str] = set() + + +class InstallBusy(RuntimeError): + pass + + +def assert_not_running() -> None: + with _install_lock: + if hermes_home_key() in _running: + raise InstallBusy("an install is already running for this profile") + + +def install_packages(*, ask_password: Callable[[], str], on_line: Callable[[str], None], + timeout_seconds: float = 900.0) -> int: + """Run the package install; returns the process exit code (0 = success, ``-1`` = cancelled).""" + if not runtime.is_supported_host(): + raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") + cmd = runtime.install_command() + if cmd is None: + raise RuntimeError("no supported package manager (apt-get, dnf, pacman) found on this host") + key = hermes_home_key() + with _install_lock: + if key in _running: + raise InstallBusy("an install is already running for this profile") + _running.add(key) + try: + return _run(cmd, ask_password=ask_password, on_line=on_line, timeout_seconds=timeout_seconds) + finally: + with _install_lock: + _running.discard(key) + + +def _sudo_nopasswd() -> bool: + try: + return subprocess.run(["sudo", "-n", "true"], capture_output=True, timeout=3, + stdin=subprocess.DEVNULL).returncode == 0 + except Exception: + return False + + +def _run(cmd: str, *, ask_password: Callable[[], str], on_line: Callable[[str], None], + timeout_seconds: float) -> int: + argv = shlex.split(cmd) + assert argv[0] == "sudo", cmd + stdin_payload: Optional[str] = None + if not _sudo_nopasswd(): + password = ask_password() or "" + if not password: + on_line("install cancelled: no sudo password provided") + return -1 + # -S: read the password from stdin; -p '': no prompt text mixed into the streamed output. + argv = ["sudo", "-S", "-p", "", *argv[1:]] + stdin_payload = password + "\n" + on_line(f"$ {cmd}") + env = {"DEBIAN_FRONTEND": "noninteractive", "LC_ALL": "C.UTF-8"} + proc = subprocess.Popen( # windows-footgun: ok — Linux-only (is_supported_host) + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env={**os.environ, **env}, text=True, encoding="utf-8", errors="replace", start_new_session=True) + try: + if stdin_payload is not None: + proc.stdin.write(stdin_payload) # type: ignore[union-attr] + proc.stdin.close() # type: ignore[union-attr] + except OSError: + pass + timer = threading.Timer(timeout_seconds, proc.kill) + timer.start() + try: + for line in proc.stdout: # type: ignore[union-attr] + on_line(line.rstrip("\n")) + return proc.wait() + finally: + timer.cancel() diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index 9f9aede01000..7e853e0f7e1e 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -3,7 +3,10 @@ ``display.status`` reports runtime + lease; ``display.start`` / ``display.stop`` manage the Xvnc/Xfce process; ``display.observe`` mints a single-use ticket the renderer redeems on ``/api/display/ws`` (``hermes_cli.web_routers.display``) to stream raw RFB; ``display.lease.acquire`` / ``release`` are -Take over / Hand back. Every handler is profile-scoped so a multiplexed gateway answers for the bot +Take over / Hand back. ``display.install`` runs the distro package install on the gateway host: sudo +privilege is asked for through the masked ``display.install.sudo.request`` card (same ``_block`` bridge as +the terminal tool's sudo prompt), stdout streams as ``display.install.log`` and the run ends with +``display.install.done`` carrying a fresh status snapshot. Every handler is profile-scoped so a multiplexed gateway answers for the bot the pane is looking at. Lease transitions fan out as the global ``display.lease`` event so every connected client repaints (badge on the bot row, red border on the viewer, agent handoff prompt). @@ -97,6 +100,49 @@ def _(rid, params: dict) -> dict: return _err(rid, _DISPLAY_ERR, str(e)) +@method("display.install") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Start the package install in the background; the renderer follows ``display.install.log`` / + ``display.install.done``. Refused while one is already running for this profile.""" + from hermes_constants import hermes_home_key + from tools.bot_desktop import install as _bd_install, runtime as _bd_runtime + if not _bd_runtime.is_supported_host(): + return _err(rid, _DISPLAY_ERR, "Bot Desktop runs on Linux gateway hosts only") + if _bd_runtime.install_command() is None: + return _err(rid, _DISPLAY_ERR, "no supported package manager (apt-get, dnf, pacman) on this host") + profile_key = hermes_home_key() + sid = str(params.get("session_id") or "") + + def _ask_password() -> str: + return _block("display.install.sudo.request", sid, {"profile_key": profile_key}, timeout=300) + + def _line(text: str) -> None: + _broadcast_global_event("display.install.log", {"profile_key": profile_key, "line": text}) + + # The worker thread has no context-bound transport; the sudo card must reach the CLIENT that + # clicked Install, so the caller's transport is carried across. + from .transport import bind_transport, current_transport + caller_transport = current_transport() + + def _run() -> None: + bind_transport(caller_transport) + try: + code = _bd_install.install_packages(ask_password=_ask_password, on_line=_line) + except Exception as e: + _line(f"install failed: {e}") + code = 1 + _broadcast_global_event("display.install.done", {"profile_key": profile_key, "code": code, + "status": _bd_runtime.status().as_dict()}) + + try: + _bd_install.assert_not_running() + except _bd_install.InstallBusy as e: + return _err(rid, _DISPLAY_ERR, str(e)) + threading.Thread(target=_run, name=f"bot-desktop-install:{profile_key}", daemon=True).start() + return _ok(rid, {"started": True, "command": _bd_runtime.install_command(), "profile_key": profile_key}) + + @method("display.lease.acquire") @_profile_scoped def _(rid, params: dict) -> dict: diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index bdb0a40b929f..8824dc3b23c6 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -1113,7 +1113,7 @@ def _(rid, params: dict) -> dict: _LATE_RESPOND_KEYS = { "terminal.read.respond": "text", "preview.read.respond": "text", "preview.act.respond": "text", "window.read.respond": "text", "tour.respond": "text", "mcp.setup.respond": "result", - "sudo.respond": "password", "secret.respond": "value", "vault.unlock.respond": "password", + "sudo.respond": "password", "display.install.sudo.respond": "password", "secret.respond": "value", "vault.unlock.respond": "password", "vault.save_login.respond": "login", "vault.code.respond": "code"} for _name, _key in _LATE_RESPOND_KEYS.items(): method(_name)(lambda rid, params, _k=_key: _respond(rid, params, _k, allow_expired=True)) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index bb99b563fa76..dcc55c923f3f 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -1252,7 +1252,7 @@ def _enable_gateway_prompts() -> None: # Blocking bridges whose `*.respond` tolerates a late reply (allow_expired=True): on timeout the tool # returns empty, but a slow renderer could still answer and hit a raw 4009 — `.expire` tears the card down. _EXPIRING_REQUESTS = frozenset({ - "secret.request", "sudo.request", "vault.unlock.request", "vault.save_login.request", "vault.code.request", "clarify.request", + "secret.request", "sudo.request", "display.install.sudo.request", "vault.unlock.request", "vault.save_login.request", "vault.code.request", "clarify.request", "terminal.read.request", "preview.read.request", "preview.act.request", "window.read.request", "mcp.setup.request", "tour.request", diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md index 6079927774cc..37c2db1a72dc 100644 --- a/website/docs/user-guide/features/bot-screen.md +++ b/website/docs/user-guide/features/bot-screen.md @@ -23,9 +23,13 @@ hosted-agent products). - The gateway host runs Linux. macOS and Windows hosts already have a real display; the pane is not offered there. - TigerVNC's `Xvnc` and the Xfce core components are installed on the host. - Hermes Desktop and `hermes computer-use screen status` print the exact - package-manager line when they are missing; `hermes computer-use screen - install` runs it for you: + Nothing installs them silently: `hermes update` and fresh installs leave every + machine as it is. When they are missing the Screen pane in Hermes Desktop shows + **Install on host** — one click runs the package manager on the gateway host + (it asks for that host's sudo password in a masked card; the password goes to + that host only and is never stored) and streams the log. From a shell, + `hermes computer-use screen status` prints the exact line and + `hermes computer-use screen install` runs it: | Distro | Packages | |---|---| @@ -39,7 +43,17 @@ hosted-agent products). ## Using it -1. In Hermes Desktop open **Bots**, right-click a bot, choose **Open Screen**. +Every bot's computer is one click away in three places of Hermes Desktop: + +- **Bots → a bot → Scheduled Jobs**: the **Screen** box at the top of the pane + (above the routines) shows whether the screen is running and who holds + control; click it to open. +- **Bots → right-click a bot → Open Screen**. +- **Sessions sidebar**, grouped by gateway / profile: the same **Screen** box + sits under each profile's header, so a profile's machine is reachable from + its conversations too. + +1. Open the Screen with any of the entries above. The first time, click **Start screen** (or leave `bot_desktop.auto_start` on, the default: the screen starts on the bot's first `computer_use` call when the host has no display). A headed browser opens on the screen once it is running. @@ -111,8 +125,9 @@ Xauthority, launcher log, per-profile xfconf). ## Troubleshooting -- **"Screen packages missing"** — run the printed install line on the gateway - host (not on the machine running Hermes Desktop). +- **"Screen packages missing"** — click **Install on host** in the pane, or run + the printed install line on the gateway host (not on the machine running + Hermes Desktop). The pane refuses a second install while one is running. - **Screen starts then stops** — read `/bot-desktop/launcher.log`. - **Typing produces wrong characters** — the screen uses a US keymap so RFB keysyms and cua-driver agree; change it with `setxkbmap` on that `DISPLAY` From 52c93215f440bbf0ed70d71d06213fdeaa2cfc16 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 06:37:50 -0700 Subject: [PATCH 05/55] feat(bot-screen): live desktop preview as the hero of a bot's Scheduled Jobs pane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bot's computer is now the first thing in its pane: a big 16:10 box above the title that shows an actual picture of the desktop, refreshed every 4 s while the hero is on screen (paused when scrolled away or the window is hidden). Caption carries who holds control; the chip reads Open live / Start / Install. Clicking the picture expands into the live Screen pane where the user can take over. - `display.thumbnail` RPC: one JPEG grab of the profile's Xvnc display via Pillow's ImageGrab (XAUTHORITY from the launcher's published env), scaled to <=960x600, returned as a data URL. Read-only: it never touches the lease, so a human in control is not disturbed and the bot is not blocked. - `ScreenHero` replaces the small portal row in the routines pane; the compact `ScreenPortal` stays for the Sessions sidebar group header where a 16:10 box would crowd the list. Live: hero "Screen is off" → click → pane → Start → hero shows the Xfce desktop with "Live · bot in control"; an xmessage window opened on the bot's display appeared in the hero on the next refresh; clicking the hero opened the live pane (canvas + Take over). --- apps/desktop/src/plugins/hermes-bots/cron.tsx | 8 +- apps/desktop/src/plugins/hermes-bots/i18n.ts | 30 ++++ .../src/plugins/hermes-bots/screen-hero.tsx | 147 ++++++++++++++++++ tests/tools/test_bot_desktop_thumbnail.py | 14 ++ tools/bot_desktop/thumbnail.py | 41 +++++ tui_gateway/methods_display.py | 11 ++ .../docs/user-guide/features/bot-screen.md | 8 +- 7 files changed, 252 insertions(+), 7 deletions(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-hero.tsx create mode 100644 tests/tools/test_bot_desktop_thumbnail.py create mode 100644 tools/bot_desktop/thumbnail.py diff --git a/apps/desktop/src/plugins/hermes-bots/cron.tsx b/apps/desktop/src/plugins/hermes-bots/cron.tsx index 74ec2deb2aee..c786f276d8f0 100644 --- a/apps/desktop/src/plugins/hermes-bots/cron.tsx +++ b/apps/desktop/src/plugins/hermes-bots/cron.tsx @@ -45,7 +45,7 @@ import { labeled } from './dialog-parts' import { botsText, useBots } from './i18n' import { displayName } from './labels' import { botConnectionRoute, botRosterMeta, requestForBot } from './routing' -import { ScreenPortal } from './screen-portal' +import { ScreenHero } from './screen-hero' import { ID } from './shared' import type { BotMeta, RosterRow, RoutineJob } from './types' @@ -1241,6 +1241,9 @@ export function RoutinesPane() { return (
+
+ +
@@ -1266,9 +1269,6 @@ export function RoutinesPane() {
-
- -
{staleNotice ? (
{staleNotice} diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index 43550dd1ddc5..5eef7a47735d 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -239,6 +239,12 @@ type BotsMessages = { noPackageManager: string portalTitle: string portalOpen: string + heroStopped: string + heroNotInstalled: string + heroConnecting: string + heroOpenLive: string + heroInstall: string + heroStart: string portalWatching: string portalYouControl: string portalOtherControls: string @@ -499,6 +505,12 @@ const en: BotsMessages = { noPackageManager: 'No supported package manager (apt, dnf, pacman) was found on the gateway host.', portalTitle: 'Screen', portalOpen: 'Open', + heroStopped: 'Screen is off', + heroNotInstalled: 'Not installed on this host', + heroConnecting: 'Checking the screen…', + heroOpenLive: 'Open live', + heroInstall: 'Install', + heroStart: 'Start', portalWatching: 'Live · bot in control', portalYouControl: 'Live · you are in control', portalOtherControls: 'Live · another viewer in control', @@ -754,6 +766,12 @@ const ja: BotsMessages = { noPackageManager: 'ゲートウェイホストに対応するパッケージマネージャー (apt, dnf, pacman) が見つかりません。', portalTitle: 'スクリーン', portalOpen: '開く', + heroStopped: '画面は停止中', + heroNotInstalled: 'このホストには未インストール', + heroConnecting: '画面を確認中…', + heroOpenLive: 'ライブで開く', + heroInstall: 'インストール', + heroStart: '開始', portalWatching: 'ライブ · ボットが操作中', portalYouControl: 'ライブ · あなたが操作中', portalOtherControls: 'ライブ · 別のビューアーが操作中', @@ -1004,6 +1022,12 @@ const zh: BotsMessages = { noPackageManager: '网关主机上未找到受支持的包管理器（apt、dnf、pacman）。', portalTitle: '屏幕', portalOpen: '打开', + heroStopped: '屏幕已关闭', + heroNotInstalled: '此主机未安装', + heroConnecting: '正在检查屏幕…', + heroOpenLive: '实时打开', + heroInstall: '安装', + heroStart: '启动', portalWatching: '直播 · 机器人控制中', portalYouControl: '直播 · 你在控制', portalOtherControls: '直播 · 其他查看者控制中', @@ -1254,6 +1278,12 @@ const zhHant: BotsMessages = { noPackageManager: '閘道主機上找不到受支援的套件管理器（apt、dnf、pacman）。', portalTitle: '螢幕', portalOpen: '開啟', + heroStopped: '螢幕已關閉', + heroNotInstalled: '此主機未安裝', + heroConnecting: '正在檢查螢幕…', + heroOpenLive: '即時開啟', + heroInstall: '安裝', + heroStart: '啟動', portalWatching: '直播 · 機器人控制中', portalYouControl: '直播 · 您在控制', portalOtherControls: '直播 · 其他檢視者控制中', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx new file mode 100644 index 000000000000..1154668df7f2 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx @@ -0,0 +1,147 @@ +/** + * Screen hero — the big preview of a bot's computer at the top of its pane. + * + * Shows a live thumbnail (one `display.thumbnail` JPEG every few seconds while + * the screen runs and the hero is on screen), or an honest placeholder for + * stopped / not installed / unsupported. Clicking it opens the full Screen pane + * where the user can take over. + */ + +import { Codicon } from '@hermes/plugin-sdk' +import { useEffect, useRef, useState } from 'react' + +import { useBots } from './i18n' +import { displayRequest } from './screen-connection' +import { openBotScreen } from './screen-open' +import { type PortalTone, useScreenPortalState } from './screen-portal' +import type { BotMeta, RosterRow } from './types' + +const REFRESH_MS = 4000 + +function useLiveThumbnail(bot: RosterRow, running: boolean) { + const [dataUrl, setDataUrl] = useState(null) + const boxRef = useRef(null) + + useEffect(() => { + if (!running) { + setDataUrl(null) + + return + } + + let cancelled = false + let timer: number | null = null + let visible = true + + const observer = + typeof IntersectionObserver === 'undefined' + ? null + : new IntersectionObserver(entries => { + visible = entries.some(entry => entry.isIntersecting) + }) + + if (observer && boxRef.current) { + observer.observe(boxRef.current) + } + + const tick = () => { + if (cancelled) { + return + } + + if (!visible || document.hidden) { + timer = window.setTimeout(tick, REFRESH_MS) + + return + } + + void displayRequest<{ data_url: string | null }>(bot, 'display.thumbnail') + .then(result => { + if (!cancelled) { + setDataUrl(result.data_url) + } + }) + .catch(() => { + /* transient: the next tick retries; the last good frame stays up */ + }) + .finally(() => { + if (!cancelled) { + timer = window.setTimeout(tick, REFRESH_MS) + } + }) + } + + tick() + + return () => { + cancelled = true + observer?.disconnect() + + if (timer !== null) { + window.clearTimeout(timer) + } + } + }, [bot, running]) + + return { dataUrl, boxRef } +} + +const TONE_RING: Partial> = { + human: 'ring-2 ring-red-500/80', + other: 'ring-2 ring-amber-500/70' +} + +export function ScreenHero({ bot, meta }: { bot: RosterRow; meta?: BotMeta | null }) { + const t = useBots() + const { tone } = useScreenPortalState(bot) + const running = tone === 'live' || tone === 'human' || tone === 'other' + const { dataUrl, boxRef } = useLiveThumbnail(bot, running) + + if (tone === 'unsupported') { + return null + } + + const caption = { + live: t.screen.portalWatching, + human: t.screen.portalYouControl, + other: t.screen.portalOtherControls, + off: t.screen.heroStopped, + missing: t.screen.heroNotInstalled, + unsupported: t.screen.portalUnsupported, + unknown: t.screen.heroConnecting + }[tone] + + const cta = running ? t.screen.heroOpenLive : tone === 'missing' ? t.screen.heroInstall : tone === 'off' ? t.screen.heroStart : '' + + return ( + + ) +} diff --git a/tests/tools/test_bot_desktop_thumbnail.py b/tests/tools/test_bot_desktop_thumbnail.py new file mode 100644 index 000000000000..97865c3ce72a --- /dev/null +++ b/tests/tools/test_bot_desktop_thumbnail.py @@ -0,0 +1,14 @@ +"""Bot Desktop thumbnail: a stopped screen yields no frame and never touches X.""" + +from __future__ import annotations + +import sys + +from tools.bot_desktop import runtime, thumbnail + + +def test_no_running_screen_returns_none_without_grabbing(monkeypatch): + monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":99"}) + monkeypatch.setattr(runtime, "_launcher_pid", lambda: None) + monkeypatch.setitem(sys.modules, "PIL.ImageGrab", None) # an import would now fail loudly + assert thumbnail.thumbnail_data_url() is None diff --git a/tools/bot_desktop/thumbnail.py b/tools/bot_desktop/thumbnail.py new file mode 100644 index 000000000000..ee41e6ffbc9b --- /dev/null +++ b/tools/bot_desktop/thumbnail.py @@ -0,0 +1,41 @@ +"""Thumbnail of a bot's screen: one JPEG grab of the profile's Xvnc display. + +Feeds the Screen hero in Hermes Desktop (the big preview at the top of a bot's pane). Read-only: +it never touches the lease, so a human in control is not disturbed and the bot is not blocked. +""" + +from __future__ import annotations + +import base64 +import io +import os +from typing import Optional + +from tools.bot_desktop import runtime + +THUMB_MAX = (960, 600) + + +def thumbnail_data_url(max_size: tuple[int, int] = THUMB_MAX, quality: int = 72) -> Optional[str]: + """``data:image/jpeg;base64,...`` of the running screen, or ``None`` when no screen is up.""" + env = runtime.published_env() + display = env.get("DISPLAY") + if not display or runtime._launcher_pid() is None: + return None + from PIL import ImageGrab # Pillow is a hard dependency; import lazily to keep status calls cheap + + # Xlib reads XAUTHORITY from the process env; the launcher publishes a per-profile cookie file. + previous = os.environ.get("XAUTHORITY") + if env.get("XAUTHORITY"): + os.environ["XAUTHORITY"] = env["XAUTHORITY"] + try: + image = ImageGrab.grab(xdisplay=display) + finally: + if previous is None: + os.environ.pop("XAUTHORITY", None) + else: + os.environ["XAUTHORITY"] = previous + image.thumbnail(max_size) + buf = io.BytesIO() + image.convert("RGB").save(buf, "JPEG", quality=quality, optimize=True) + return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode("ascii") diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index 7e853e0f7e1e..043b8bc90773 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -56,6 +56,17 @@ def _(rid, params: dict) -> dict: return _err(rid, _DISPLAY_ERR, str(e)) +@method("display.thumbnail") +@_profile_scoped +def _(rid, params: dict) -> dict: + """One JPEG grab of the bot's screen (``data_url``: null while stopped). Read-only: no lease change.""" + try: + from tools.bot_desktop.thumbnail import thumbnail_data_url + return _ok(rid, {"data_url": thumbnail_data_url()}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + @method("display.start") @_profile_scoped def _(rid, params: dict) -> dict: diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md index 37c2db1a72dc..796e0179c93a 100644 --- a/website/docs/user-guide/features/bot-screen.md +++ b/website/docs/user-guide/features/bot-screen.md @@ -45,9 +45,11 @@ hosted-agent products). Every bot's computer is one click away in three places of Hermes Desktop: -- **Bots → a bot → Scheduled Jobs**: the **Screen** box at the top of the pane - (above the routines) shows whether the screen is running and who holds - control; click it to open. +- **Bots → a bot → Scheduled Jobs**: the bot's screen is the hero at the very + top of the pane, above the title and the routines: a live preview of the + desktop (refreshed every few seconds while the pane is visible) with who holds + control; click the picture to expand into live access. While the screen is off + or not installed the same box says so and offers Start / Install. - **Bots → right-click a bot → Open Screen**. - **Sessions sidebar**, grouped by gateway / profile: the same **Screen** box sits under each profile's header, so a profile's machine is reachable from From eebd158c4faf7a6e853edb4e6c1e8b261d662d86 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 08:27:39 -0700 Subject: [PATCH 06/55] =?UTF-8?q?feat(bot-screen):=20Hermes=20look=20for?= =?UTF-8?q?=20the=20bot's=20desktop=20=E2=80=94=20wallpaper,=20dark=20them?= =?UTF-8?q?e,=20curated=20dock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vendor Xfce panel layout (copied only to silence the first-run dialog) put a light grey bar with dead launchers on every bot screen: File Manager and Text Editor pointed at Thunar/Mousepad we deliberately do not install, Web Browser opened exo's "pick a browser" dialog. It read as an unconfigured VM, and so did the thumbnail in Desktop. - Wallpaper: `tools/bot_desktop/wallpaper.png` (the Nous gradient), seeded via xfconf as a zoomed backdrop over the existing colour fallback. - Dark theme: first dark GTK theme the host ships (Adwaita-dark, Breeze-Dark, Greybird-dark, Arc-Dark, else Adwaita), dark icon theme likewise, xfwm4 Default decorations, DejaVu fonts; both panels dark with slight translucency. - Own panel layout: top bar = menu · task list · tray · clock; bottom dock = only launchers whose program exists on this host, the browser pinned to the one the bot drives (chrome → chromium → firefox) so a human who takes over lands in the bot's own browser profile. Anything the user installs still appears in the Applications menu; the dock is the only curated part. - `launcher.sh` and the wallpaper declared as package data (the launcher was already missing from sealed wheels). - `HERMES_BD_SEED_ONLY=1` stops the launcher after seeding so the config tree is testable on a fake PATH without an X server. Live: fresh screen shows the gradient, dark panels, two dock icons; XTEST clicks on the dock opened xfce4-terminal and Chrome, both dark-themed and listed in the task bar; the Desktop hero and Screen pane show the same picture. --- pyproject.toml | 3 + tests/tools/test_bot_desktop_launcher_seed.py | 62 ++++++++++ tools/bot_desktop/launcher.sh | 110 +++++++++++++++++- tools/bot_desktop/wallpaper.png | Bin 0 -> 2475 bytes 4 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 tests/tools/test_bot_desktop_launcher_seed.py create mode 100644 tools/bot_desktop/wallpaper.png diff --git a/pyproject.toml b/pyproject.toml index ee77c1e53a9b..7deda29005ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -578,6 +578,9 @@ gateway = ["assets/**/*"] # sealed wheels with the plugin Python modules; without this declaration the # wheel contains adapters but discovery finds zero bundled plugins. plugins = ["**/plugin.yaml", "**/plugin.yml"] +# Bot Desktop starts Xvnc + Xfce through a shell launcher and seeds the wallpaper; both are read +# via Path(__file__).parent in tools/bot_desktop/runtime.py and vanish from sealed wheels otherwise. +tools = ["bot_desktop/launcher.sh", "bot_desktop/wallpaper.png"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/tools/test_bot_desktop_launcher_seed.py b/tests/tools/test_bot_desktop_launcher_seed.py new file mode 100644 index 000000000000..e404ddc5d138 --- /dev/null +++ b/tests/tools/test_bot_desktop_launcher_seed.py @@ -0,0 +1,62 @@ +"""Bot Desktop launcher seeds: the dock only points at programs that exist, the look is applied.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +LAUNCHER = Path(__file__).resolve().parents[2] / "tools" / "bot_desktop" / "launcher.sh" +pytestmark = pytest.mark.linux_only + + +def _seed(tmp_path: Path, fake_bins: list[str]) -> Path: + bindir = tmp_path / "bin" + bindir.mkdir() + for name in fake_bins: + exe = bindir / name + exe.write_text("#!/bin/sh\n", encoding="utf-8") + exe.chmod(0o755) + # The script's own tooling (mkdir, sed, cat, awk...) symlinked in, so PATH need not contain the + # host's /usr/bin where a real chrome/thunar would leak into the dock under test. + for tool in ("mkdir", "sed", "cat", "printf", "dirname", "bash", "sh", "rm", "ln", "touch", "chmod", "xauth", "od", "tr", "awk"): + real = shutil.which(tool) + if real and not (bindir / tool).exists(): + (bindir / tool).symlink_to(real) + cfg = tmp_path / "xdg" + env = { + "PATH": str(bindir), + "HOME": str(tmp_path), + "HERMES_BD_PROFILE": "t", "HERMES_BD_DISPLAY_NUM": "99", + "HERMES_BD_SOCKET": str(tmp_path / "rfb.sock"), "HERMES_BD_XAUTH": str(tmp_path / "Xauthority"), + "HERMES_BD_ENV_FILE": str(tmp_path / "env"), "HERMES_BD_CONFIG_HOME": str(cfg), + "HERMES_BD_SEED_ONLY": "1", + } + subprocess.run(["bash", str(LAUNCHER)], env=env, check=True, stdin=subprocess.DEVNULL, capture_output=True, timeout=30) + return cfg + + +def test_dock_lists_only_programs_present_on_path(tmp_path): + cfg = _seed(tmp_path, ["xfce4-terminal", "firefox"]) # no thunar, no mousepad, no chrome + panel = ET.parse(cfg / "xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml") # well-formed or this raises + launcher_ids = [str(p.get("name")) for p in panel.iter("property") if p.get("value") == "launcher"] + execs = sorted( + line.split("=", 1)[1] + for pid in launcher_ids + for line in (cfg / "xfce4/panel" / pid.replace("plugin-", "launcher-") / "hermes.desktop").read_text(encoding="utf-8").splitlines() + if line.startswith("Exec=") + ) + assert execs == ["firefox", "xfce4-terminal"] + + +def test_look_is_seeded_with_wallpaper_and_theme(tmp_path): + cfg = _seed(tmp_path, ["xfce4-terminal"]) + desktop = (cfg / "xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml").read_text(encoding="utf-8") + xsettings = (cfg / "xfce4/xfconf/xfce-perchannel-xml/xsettings.xml").read_text(encoding="utf-8") + assert str(LAUNCHER.with_name("wallpaper.png")) in desktop + assert "PLACEHOLDER" not in desktop + xsettings + assert os.path.isfile(LAUNCHER.with_name("wallpaper.png")) diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh index 60c3635982c7..2f7348aa892b 100755 --- a/tools/bot_desktop/launcher.sh +++ b/tools/bot_desktop/launcher.sh @@ -45,6 +45,14 @@ rm -f "$HERMES_BD_SOCKET" "/tmp/.X${HERMES_BD_DISPLAY_NUM}-lock" "/tmp/.X11-unix : > "$XAUTHORITY"; chmod 600 "$XAUTHORITY" xauth -q -f "$XAUTHORITY" add "$DISPLAY" MIT-MAGIC-COOKIE-1 "$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" +# ---- look: dark theme from whatever the host ships (first match wins), Hermes wallpaper ---- +pick_theme() { local d t; for t in "$@"; do for d in /usr/share/themes "$HOME/.themes"; do [[ -d "$d/$t" ]] && { echo "$t"; return; }; done; done; echo "$1"; } +pick_icons() { local d t; for t in "$@"; do for d in /usr/share/icons "$HOME/.icons"; do [[ -d "$d/$t" ]] && { echo "$t"; return; }; done; done; echo "$1"; } +GTK_THEME_NAME=$(pick_theme Adwaita-dark Breeze-Dark Greybird-dark Arc-Dark Adwaita) +WM_THEME_NAME=$(pick_theme Default-hdpi Default) # xfwm4 window themes ship with xfwm4 itself +ICON_THEME_NAME=$(pick_icons Papirus-Dark breeze-dark Adwaita hicolor) +: "${HERMES_BD_WALLPAPER:="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/wallpaper.png"}" + # ---- pre-seed xfconf BEFORE xfconfd starts (it caches; edits after start are overwritten) ---- X="$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" [[ -e "$X/xfwm4.xml" ]] || cat > "$X/xfwm4.xml" <<'EOF' @@ -54,9 +62,12 @@ X="$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" + + EOF +sed -i "s|HERMES_BD_WM_THEME|$WM_THEME_NAME|" "$X/xfwm4.xml" [[ -e "$X/xfce4-screensaver.xml" ]] || cat > "$X/xfce4-screensaver.xml" <<'EOF' @@ -69,6 +80,12 @@ EOF + + + + + + @@ -77,6 +94,7 @@ EOF EOF +sed -i "s|HERMES_BD_GTK_THEME|$GTK_THEME_NAME|; s|HERMES_BD_ICON_THEME|$ICON_THEME_NAME|" "$X/xsettings.xml" [[ -e "$X/xfce4-desktop.xml" ]] || cat > "$X/xfce4-desktop.xml" <<'EOF' @@ -85,7 +103,8 @@ EOF - + + @@ -99,12 +118,91 @@ EOF EOF -# The vendor default panel layout suppresses the first-run "Welcome to the panel" dialog. +sed -i "s|HERMES_BD_WALLPAPER_PLACEHOLDER|$HERMES_BD_WALLPAPER|" "$X/xfce4-desktop.xml" +# Own panel layout (a layout on disk also suppresses the first-run "Welcome to the panel" dialog): +# top bar = menu · tasks · tray · clock; bottom dock = only launchers whose program exists on this +# host, the browser pinned to the one the bot drives so a human lands in the bot's own browser profile. if [[ ! -e "$X/xfce4-panel.xml" ]]; then - for d in /etc/xdg/xfce4/panel/default.xml /usr/share/xfce4-panel/default.xml \ - /etc/xdg/xdg-xubuntu/xfce4/panel/default.xml; do - [[ -e "$d" ]] && { cp "$d" "$X/xfce4-panel.xml"; break; } + L="$XDG_CONFIG_HOME/xfce4/panel"; mkdir -p "$L" + dock_ids=(); n=20 + add_launcher() { # name icon exec — skipped when the executable is missing + local exe; exe=${3%% *} + command -v "$exe" >/dev/null 2>&1 || return 0 + n=$((n+1)); mkdir -p "$L/launcher-$n" + printf '[Desktop Entry]\nVersion=1.0\nType=Application\nName=%s\nIcon=%s\nExec=%s\nTerminal=false\nStartupNotify=false\n' \ + "$1" "$2" "$3" > "$L/launcher-$n/hermes.desktop" + dock_ids+=("$n") + } + add_launcher "Terminal" utilities-terminal "xfce4-terminal" + for b in google-chrome chromium chromium-browser firefox; do + command -v "$b" >/dev/null 2>&1 && { add_launcher "Browser" internet-web-browser "$b"; break; } done + add_launcher "Files" system-file-manager "thunar" + add_launcher "Text Editor" accessories-text-editor "mousepad" + dock_plugins=""; dock_items="" + for id in "${dock_ids[@]}"; do + dock_plugins+="" + dock_items+="" + done + cat > "$X/xfce4-panel.xml" < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${dock_plugins} + + + + + + + + + + + + + + + + + + + ${dock_items} + + +PANEL fi # Mask system autostarts that want logind/polkit/keyring/at-spi. for a in xfce4-screensaver light-locker xfce4-power-manager xfce-polkit \ @@ -113,6 +211,8 @@ for a in xfce4-screensaver light-locker xfce4-power-manager xfce-polkit \ [[ -e "$XDG_CONFIG_HOME/autostart/$a.desktop" ]] || \ printf '[Desktop Entry]\nType=Application\nName=%s\nHidden=true\n' "$a" > "$XDG_CONFIG_HOME/autostart/$a.desktop" done +# Tests seed the config tree on a fake PATH and stop here (no X server needed). +[[ -n "${HERMES_BD_SEED_ONLY:-}" ]] && exit 0 # ---- X server + RFB (TigerVNC Xvnc), Unix socket only ---- # SecurityTypes None is safe ONLY because -rfbport -1 disables TCP and the 0600 socket is reachable diff --git a/tools/bot_desktop/wallpaper.png b/tools/bot_desktop/wallpaper.png new file mode 100644 index 0000000000000000000000000000000000000000..3d933f827d28293629460d33587ad027ec753244 GIT binary patch literal 2475 zcmZYBNp4+57zJPlk;Hy}@AiG)cdzekCo@nGcoHGR0$2khMo2JY53C_W){reAv6sWC zy7xH_42~^%;!mfl{{Q~*{rhhhvtV|0b+vf>?%fakoiWDsEB<`@@gM%-=I6(EU;X&Y zi>u$Re#MogyO1$l*dnF}$MoS^0X#c^??edP7?GDC4pJmxjx;WiCl!jUMp?9|st$EC zpy`%qhZVZj2Hj?he!Ii4-(xr&Fr1E9o=;fbiJY;#A2TXU-GM126o`CM_>&@(ija!v znIcO|>kd`Zqh=QkyAX<&U39x0Ix2c9dUhcc1G^ab-(?;y3WI`6Ow)ykWl>`LaHw#p z@I0Z25weSjia6n^u#1d}oQi^qvh7jzOVrC1n)MpZc8gXh4tsP|bmt@bJLdj`{z2p{ zGj^dFHgp*Wx=E3cSd`e5IFz`QP~lM#gb6|_A}ZpnM4DH~sL1OU1r=pKprWFtqTX)M z?6+u+d$cF!ObHbo6&)4b8<9&FDg`1!qfsFxRG3s)RM=EFUI^Exgo+?a5mFIl1(Kpd zTGhy!7J1j97zUK9CCbej)oz2Diu$-ibKax5JD|BgqJ3Z}xMb`?F)WCvBIYEq*ojSv zLy1d?M+p^f7{jL`NHT;}L`8{&inM8vQIQWl3Mz`t3gwQH!xj}4H5D}#^*tq2G;f%< zCz(;P&`pRuY5phiVNqfGA)FwF7bWni2r~9TMO;-#s7SjO85Q|zkcxte@~}a9q~yFq zbw>#mH5D}#^;?nC^hOrisF0FL^QjXT2Kz8CeFS44Ud%pH_K~xXvO-+fNZJ-@-yx$S z->?rViv6f4Cq+d?^}tY3NyTwe%vA#-3r&QbP@?LVyeCh0M4pT|72Xe?%B7F+sSlxG zA40KYAL}LZ?TQLY*hP8Tp}b=s_xnjPNvIi5WUlBCS%^@hs8mp+Q6k=s!5eGx<%*m7 z{FeIcBTRWJIZuUs{P$FH-i_y{@~MwWB0D);6mty=86C=kQ4FXG6)F`P6=I_C_1cs; zj(D}4AG{yqTj8taEsgnB##_p}!9MaO?*{uQw!9nSRKzW1AD8 zYi_|-s8C&ax(DB+!j2Ge&&K|=W7$WRRmclY^QxVu`Lj8?;T+v@j_xUt63)_Trj82r zr2?5(Dl|raO~o}8H&o0_*tZs(g$-AsLh}$90YWQ89lB`tT03?-A|O9h$;HQGxmUTH8WF)6Vqv1e4wZFma@zDk8o zg&88YV Date: Sat, 12 Sep 2026 09:13:29 -0700 Subject: [PATCH 07/55] fix(bot-screen): lease shared across processes, takeover fences in-flight actions, sudo reply pinned to origin, dock Browser is the bot's browser, safe display reuse, install keeps profile scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent review of the PR head found the control boundary only held inside one process and several claims the code did not back. Each item below was reproduced, fixed, covered by an invariant test proven red without the fix, and re-verified live on a real Xvnc/Xfce screen. - Lease authority on disk. `lease.json` under an fcntl lock in the profile's bot-desktop dir; every read goes to the file. `hermes serve` (viewer bridge), the messaging gateway, a CLI turn and isolated workers now agree. Live: a takeover in process A made `computer_use capture` in process B return human_has_control; release in C made B work again. - Takeover fences admitted actions. `handle_computer_use` re-checks the lease under the dispatch lock and discards a result produced after the lease epoch changed, so an action admitted before a takeover cannot picture what the human typed during approval / backend start-up waits. - Sudo reply pinned to its origin. `SudoRequest.origin` records the (connection, profile) the card came from; SudoDialog answers through `requestGatewayForAgent` on that socket, never the foreground gateway. A password typed for host A can no longer reach host B. `sudo.expire` and `display.install.sudo.expire` now tear the card down (the Desktop never handled sudo.expire). - Dock Browser IS the bot's browser. `tools/bot_desktop/browser.py` resolves one identity — the Chromium agent-browser drives + a persistent per-profile user-data-dir (`bot-desktop/browser-profile`) — and both sides use it: the agent env gets AGENT_BROWSER_EXECUTABLE_PATH / AGENT_BROWSER_PROFILE, the dock launcher gets the same exe + --user-data-dir. Live: the bot wrote localStorage on http://127.0.0.1:8765 through agent-browser; a dock click and a typed URL on the screen showed BOT-WROTE-THIS in Chrome for Testing. - Safe display reuse. Allocation under a host-wide lock; a recorded number is reused only when no live server holds it; the launcher never unlinks a lock whose pid is alive. Live: A stopped, B took :20, A restarted on :21, B kept running. - Install worker keeps the caller's profile scope (copy_context carries the HERMES_HOME override and the transport); the done event carries the requested profile's status. - Honest scope: `bot_desktop.auto_start` defaults to false (opt-in; Start lives in the Screen pane); request_handoff no longer claims a Telegram/Discord message was sent — the model relays the ask in its reply; docs match. --- .../gateway-event/input-requests.ts | 11 ++ .../src/components/prompt-overlays.tsx | 14 +- apps/desktop/src/store/prompts.ts | 4 + hermes_cli/config_defaults.py | 6 +- hermes_cli/web_routers/display.py | 11 +- tests/tools/test_bot_desktop_browser.py | 26 ++++ tests/tools/test_bot_desktop_launcher_seed.py | 8 +- tests/tools/test_bot_desktop_lease.py | 35 +++++ tests/tools/test_bot_desktop_runtime.py | 29 ++++ tests/tools/test_bot_desktop_thumbnail.py | 14 -- tests/tui_gateway/test_display_methods.py | 33 ++++ tools/bot_desktop/browser.py | 58 +++++++ tools/bot_desktop/launcher.sh | 16 +- tools/bot_desktop/lease.py | 146 ++++++++++++------ tools/bot_desktop/runtime.py | 40 +++-- tools/computer_use/handoff.py | 10 +- tools/computer_use/tool.py | 20 ++- tui_gateway/methods_display.py | 14 +- .../docs/user-guide/features/bot-screen.md | 23 +-- 19 files changed, 407 insertions(+), 111 deletions(-) create mode 100644 tests/tools/test_bot_desktop_browser.py create mode 100644 tests/tools/test_bot_desktop_runtime.py delete mode 100644 tests/tools/test_bot_desktop_thumbnail.py create mode 100644 tests/tui_gateway/test_display_methods.py create mode 100644 tools/bot_desktop/browser.py diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts index e6397a32b84f..ac0ab916f65e 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts @@ -12,12 +12,14 @@ import { import { $gateway } from '@/store/gateway' import { setMcpSetupRequest } from '@/store/mcp-setup' import { dispatchNativeNotification } from '@/store/native-notifications' +import { $activeGatewayProfile } from '@/store/profile' import { $vaultCodeRequests, $vaultSaveLoginRequests, $vaultUnlockRequests, clearVaultCodeRequest, clearVaultSaveLoginRequest, + clearSudoRequest, clearVaultUnlockRequest, receiveApprovalRequest, setSecretRequest, @@ -203,6 +205,14 @@ export function handleInputRequestEvent(ctx: GatewayEventContext): boolean { return true } + if (event.type === 'sudo.expire' || event.type === 'display.install.sudo.expire') { + // The backend gave up waiting; tear the card down so a late Send cannot go anywhere. + const requestId = typeof payload?.request_id === 'string' ? payload.request_id : '' + clearSudoRequest(sessionId ?? undefined, requestId || undefined) + + return true + } + if (event.type === 'clarify.expire') { if (!sessionId) { return true @@ -324,6 +334,7 @@ export function handleInputRequestEvent(ctx: GatewayEventContext): boolean { setSudoRequest({ requestId, sessionId: sessionId ?? null, + origin: { connectionId: event.connectionId ?? null, profile: event.profile ?? $activeGatewayProfile.get() }, ...(install ? { respondMethod: 'display.install.sudo.respond', description: translateNow('prompts.sudoInstallDesc') } : {}) }) diff --git a/apps/desktop/src/components/prompt-overlays.tsx b/apps/desktop/src/components/prompt-overlays.tsx index 0e76b01249f9..187bd341013c 100644 --- a/apps/desktop/src/components/prompt-overlays.tsx +++ b/apps/desktop/src/components/prompt-overlays.tsx @@ -19,7 +19,7 @@ import { useI18n } from '@/i18n' import { isMissingPendingPromptRequest } from '@/lib/gateway-rpc' import { triggerHaptic } from '@/lib/haptics' import { KeyRound, Loader2, Lock, ShieldLock } from '@/lib/icons' -import { $gateway } from '@/store/gateway' +import { $gateway, requestGatewayForAgent } from '@/store/gateway' import { notifyError } from '@/store/notifications' import { clearSecretRequest, @@ -78,10 +78,14 @@ function SudoDialog({ sessionId }: { sessionId: string | null }) { setSubmitting(true) try { - await gateway.request<{ status?: string }>(request.respondMethod ?? 'sudo.respond', { - password: value, - request_id: request.requestId - }) + const method = request.respondMethod ?? 'sudo.respond' + const reply = { password: value, request_id: request.requestId } + // Pinned to the socket the request came from: the foreground gateway may be another host. + if (request.origin) { + await requestGatewayForAgent<{ status?: string }>(request.origin.connectionId, request.origin.profile, method, reply) + } else { + await gateway.request<{ status?: string }>(method, reply) + } triggerHaptic('submit') clearSudoRequest(request.sessionId, request.requestId) } catch (error) { diff --git a/apps/desktop/src/store/prompts.ts b/apps/desktop/src/store/prompts.ts index e57fbf660bbd..e08c45846971 100644 --- a/apps/desktop/src/store/prompts.ts +++ b/apps/desktop/src/store/prompts.ts @@ -103,6 +103,10 @@ export interface SudoRequest extends KeyedPrompt { respondMethod?: string /** Description override so the card can say WHAT the password is for. */ description?: string + /** Immutable origin of the request. The reply is sent to THIS socket, never to whichever + * connection happens to be in the foreground when the user presses Send — a password typed for + * host A must not travel to host B. */ + origin?: { connectionId: null | string; profile: string } } export interface SecretRequest extends KeyedPrompt { diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 746a88ffb393..ac8e5b69e7f9 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2246,8 +2246,10 @@ def _aux(timeout, *, reasoning_effort=True, **extra): # Desktop where a human can watch, take over (logins, 2FA, CAPTCHAs) and hand back. `hermes desktop`. "bot_desktop": { "geometry": "1440x900", - # Start the screen automatically the first time computer_use or a headed browser needs a display. - "auto_start": True, + # Opt-in: start the screen automatically the first time computer_use needs a display on a headless + # host. Off by default so installing TigerVNC for other reasons never yields a screen nobody asked + # for; Hermes Desktop's Screen pane offers Start and this toggle. + "auto_start": False, }, "computer_use": { # cua-driver's upstream PostHog telemetry defaults ON; Hermes sets diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py index a9fe07a5c3e5..4d81e24abba8 100644 --- a/hermes_cli/web_routers/display.py +++ b/hermes_cli/web_routers/display.py @@ -62,7 +62,8 @@ async def display_ws(ws: WebSocket) -> None: from pathlib import Path sock = Path(info["hermes_home"]) / "bot-desktop" / "rfb.sock" - profile_key = hermes_home_key(info["hermes_home"]) + profile_home = str(info["hermes_home"]) + profile_key = hermes_home_key(profile_home) viewer_id = str(info.get("viewer_id") or info.get("user_id") or "viewer") if not sock.exists(): await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop is not running") @@ -77,7 +78,7 @@ async def display_ws(ws: WebSocket) -> None: await ws.accept() loop = asyncio.get_running_loop() evicted = asyncio.Event() - held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_key)} + held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)} def _on_lease(key: str, lease) -> None: # A viewer that held control during this connection and lost it to ANOTHER human is kicked so @@ -91,7 +92,7 @@ def _on_lease(key: str, lease) -> None: loop.call_soon_threadsafe(evicted.set) unsubscribe = _lease.on_change(_on_lease) - rfb_filter = RfbClientFilter(lambda: _lease.viewer_may_send_input(viewer_id, profile_key=profile_key)) + rfb_filter = RfbClientFilter(lambda: _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)) async def rfb_to_ws() -> None: while True: @@ -136,8 +137,8 @@ async def watch_eviction() -> None: unsubscribe() writer.close() # Closing the viewer window hands control back; a stale holder never pins the agent out. - if _lease.viewer_may_send_input(viewer_id, profile_key=profile_key): - _lease.release(viewer_id, profile_key=profile_key) + if _lease.viewer_may_send_input(viewer_id, profile_key=profile_home): + _lease.release(viewer_id, profile_key=profile_home) try: await ws.close() except Exception: # already closed by the peer or by an eviction diff --git a/tests/tools/test_bot_desktop_browser.py b/tests/tools/test_bot_desktop_browser.py new file mode 100644 index 000000000000..b3bb6ce9e806 --- /dev/null +++ b/tests/tools/test_bot_desktop_browser.py @@ -0,0 +1,26 @@ +"""The dock's Browser and agent-browser resolve to one identity: same executable, same user-data-dir.""" + +from __future__ import annotations + +from tools.bot_desktop import browser, runtime + + +def test_dock_and_agent_share_browser_identity(tmp_path, monkeypatch): + exe = tmp_path / "chrome" + exe.write_text("#!/bin/sh\n", encoding="utf-8") + exe.chmod(0o755) + monkeypatch.setenv("AGENT_BROWSER_EXECUTABLE_PATH", str(exe)) + monkeypatch.delenv("AGENT_BROWSER_PROFILE", raising=False) + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bot-desktop") + + dock_exe, dock_profile = browser.dock_launch() + agent_env = browser.env_for_agent({}) + + assert dock_exe == agent_env["AGENT_BROWSER_EXECUTABLE_PATH"] == str(exe) + assert dock_profile == agent_env["AGENT_BROWSER_PROFILE"] == str(tmp_path / "bot-desktop" / "browser-profile") + + +def test_user_pinned_profile_wins(tmp_path, monkeypatch): + monkeypatch.setenv("AGENT_BROWSER_PROFILE", str(tmp_path / "mine")) + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bot-desktop") + assert browser.profile_dir() == tmp_path / "mine" diff --git a/tests/tools/test_bot_desktop_launcher_seed.py b/tests/tools/test_bot_desktop_launcher_seed.py index e404ddc5d138..261d9d00558d 100644 --- a/tests/tools/test_bot_desktop_launcher_seed.py +++ b/tests/tools/test_bot_desktop_launcher_seed.py @@ -14,7 +14,7 @@ pytestmark = pytest.mark.linux_only -def _seed(tmp_path: Path, fake_bins: list[str]) -> Path: +def _seed(tmp_path: Path, fake_bins: list[str], browser_exec: str = "") -> Path: bindir = tmp_path / "bin" bindir.mkdir() for name in fake_bins: @@ -35,13 +35,15 @@ def _seed(tmp_path: Path, fake_bins: list[str]) -> Path: "HERMES_BD_SOCKET": str(tmp_path / "rfb.sock"), "HERMES_BD_XAUTH": str(tmp_path / "Xauthority"), "HERMES_BD_ENV_FILE": str(tmp_path / "env"), "HERMES_BD_CONFIG_HOME": str(cfg), "HERMES_BD_SEED_ONLY": "1", + **({"HERMES_BD_BROWSER_EXEC": browser_exec} if browser_exec else {}), } subprocess.run(["bash", str(LAUNCHER)], env=env, check=True, stdin=subprocess.DEVNULL, capture_output=True, timeout=30) return cfg def test_dock_lists_only_programs_present_on_path(tmp_path): - cfg = _seed(tmp_path, ["xfce4-terminal", "firefox"]) # no thunar, no mousepad, no chrome + chrome = tmp_path / "bin" / "chrome" # the browser is the one runtime.py resolved, never a PATH scan + cfg = _seed(tmp_path, ["xfce4-terminal", "chrome", "firefox"], browser_exec=f"{chrome} --user-data-dir={tmp_path}/bp") panel = ET.parse(cfg / "xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml") # well-formed or this raises launcher_ids = [str(p.get("name")) for p in panel.iter("property") if p.get("value") == "launcher"] execs = sorted( @@ -50,7 +52,7 @@ def test_dock_lists_only_programs_present_on_path(tmp_path): for line in (cfg / "xfce4/panel" / pid.replace("plugin-", "launcher-") / "hermes.desktop").read_text(encoding="utf-8").splitlines() if line.startswith("Exec=") ) - assert execs == ["firefox", "xfce4-terminal"] + assert execs == [f"{chrome} --user-data-dir={tmp_path}/bp", "xfce4-terminal"] def test_look_is_seeded_with_wallpaper_and_theme(tmp_path): diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py index 9266ad1ec261..7a3665eb6f7f 100644 --- a/tests/tools/test_bot_desktop_lease.py +++ b/tests/tools/test_bot_desktop_lease.py @@ -65,3 +65,38 @@ def test_computer_use_refuses_every_action_while_a_human_holds_the_screen(monkey lease.release("human") done = json.loads(tool.handle_computer_use({"action": "wait_for_human", "seconds": 1})) assert done["ok"] and done["state"]["holder"] == lease.AGENT + + +def test_lease_authority_is_shared_across_processes(tmp_path): + """The gateway that streams the screen and the process running the agent are different processes; + a human takeover in one must refuse actions in the other.""" + import os + import subprocess + import sys + + lease.acquire("desktop-viewer") + probe = ("import sys; sys.path.insert(0, %r)\n" + "from tools.bot_desktop import lease\n" + "try:\n lease.assert_agent_may_act(); print('AGENT')\n" + "except lease.HumanHasControl:\n print('HUMAN')\n" + "lease.release('desktop-viewer')\n") % os.getcwd() + out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, encoding="utf-8", timeout=30, + stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": os.environ["HERMES_HOME"]}) + assert out.stdout.strip() == "HUMAN", out.stderr + assert lease.get().holder == lease.AGENT, "the other process's release is visible here" + + +def test_takeover_during_an_admitted_action_discards_its_result(monkeypatch): + """Approval / backend start-up can take seconds; a human who takes over meanwhile must not have + their keystrokes captured by an action admitted before they did.""" + from tools.computer_use import tool + + monkeypatch.setattr(tool, "_get_backend", lambda session_id="": object()) + + def _dispatch_then_takeover(backend, action, args): + lease.acquire("human") # flips while the driver call is in flight + return json.dumps({"ok": True, "action": action, "png_b64": "SECRET"}) + + monkeypatch.setattr(tool, "_dispatch", _dispatch_then_takeover) + res = json.loads(tool.handle_computer_use({"action": "capture"})) + assert res["code"] == "human_has_control" and "SECRET" not in json.dumps(res) diff --git a/tests/tools/test_bot_desktop_runtime.py b/tests/tools/test_bot_desktop_runtime.py new file mode 100644 index 000000000000..ab2080900459 --- /dev/null +++ b/tests/tools/test_bot_desktop_runtime.py @@ -0,0 +1,29 @@ +"""Bot Desktop runtime: thumbnail and display-number allocation invariants.""" + +from __future__ import annotations + +import sys + +from tools.bot_desktop import runtime, thumbnail + + +def test_no_running_screen_returns_none_without_grabbing(monkeypatch): + monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":99"}) + monkeypatch.setattr(runtime, "_launcher_pid", lambda: None) + monkeypatch.setitem(sys.modules, "PIL.ImageGrab", None) # an import would now fail loudly + assert thumbnail.thumbnail_data_url() is None + + +def test_recorded_display_held_by_a_live_server_is_not_reused(tmp_path, monkeypatch): + """After profile A stops, B may take A's number; A restarting must pick another rather than + unlink B's socket and lock.""" + import os + + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path) + (tmp_path / "display").write_text("37", encoding="utf-8") + live = {37: os.getpid()} # :37 is owned by a running server (this very process stands in for it) + monkeypatch.setattr(runtime, "_display_in_use", lambda num: num in live) + monkeypatch.setattr(runtime, "_ALLOC_LOCK", tmp_path / "alloc.lock") + assert runtime._allocate_display() != 37 + live.clear() + assert runtime._allocate_display() == 37, "a free recorded number is reclaimed" diff --git a/tests/tools/test_bot_desktop_thumbnail.py b/tests/tools/test_bot_desktop_thumbnail.py deleted file mode 100644 index 97865c3ce72a..000000000000 --- a/tests/tools/test_bot_desktop_thumbnail.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Bot Desktop thumbnail: a stopped screen yields no frame and never touches X.""" - -from __future__ import annotations - -import sys - -from tools.bot_desktop import runtime, thumbnail - - -def test_no_running_screen_returns_none_without_grabbing(monkeypatch): - monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":99"}) - monkeypatch.setattr(runtime, "_launcher_pid", lambda: None) - monkeypatch.setitem(sys.modules, "PIL.ImageGrab", None) # an import would now fail loudly - assert thumbnail.thumbnail_data_url() is None diff --git a/tests/tui_gateway/test_display_methods.py b/tests/tui_gateway/test_display_methods.py new file mode 100644 index 000000000000..d23cef507b1c --- /dev/null +++ b/tests/tui_gateway/test_display_methods.py @@ -0,0 +1,33 @@ +"""display.install runs its worker inside the caller's profile scope.""" + +from __future__ import annotations + +import threading + +import pytest + + +def test_install_worker_keeps_the_requested_profile_scope(tmp_path, monkeypatch): + from hermes_constants import get_hermes_home + from tools.bot_desktop import install, runtime + import tui_gateway.server as server + + named = tmp_path / "profiles" / "named" + named.mkdir(parents=True) + monkeypatch.setattr(server, "_profile_home", lambda name: str(named) if name == "named" else None) + monkeypatch.setattr(runtime, "is_supported_host", lambda: True) + monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y x") + seen = {} + done = threading.Event() + + def fake_install(*, ask_password, on_line, timeout_seconds=900.0): + seen["home"] = str(get_hermes_home()) + done.set() + return 0 + + monkeypatch.setattr(install, "install_packages", fake_install) + monkeypatch.setattr(server, "_broadcast_global_event", lambda *a, **k: None) + resp = server.handle_request({"jsonrpc": "2.0", "id": 1, "method": "display.install", "params": {"profile": "named"}}) + assert resp["result"]["started"], resp + assert done.wait(5) + assert seen["home"] == str(named) diff --git a/tools/bot_desktop/browser.py b/tools/bot_desktop/browser.py new file mode 100644 index 000000000000..a3fd382e0796 --- /dev/null +++ b/tools/bot_desktop/browser.py @@ -0,0 +1,58 @@ +"""The bot's browser on its Bot Desktop: one executable, one persistent user-data-dir per profile. + +The agent drives Chromium through agent-browser; a human who takes over clicks the dock's Browser +icon. Both must be THE SAME browser — same binary, same ``--user-data-dir`` — or the human logs in +to a jar the bot never sees. Chromium's singleton makes a second launch on the same user-data-dir +open a window in the running instance, which is exactly the hand-over we want. +""" + +from __future__ import annotations + +import glob +import os +import shutil +from pathlib import Path +from typing import Optional, Tuple + +from tools.bot_desktop import runtime + +_SYSTEM_BROWSERS = ("google-chrome", "google-chrome-stable", "chromium", "chromium-browser") + + +def profile_dir() -> Path: + """User-data-dir the bot's browser uses on this profile's screen (``AGENT_BROWSER_PROFILE`` wins).""" + override = os.environ.get("AGENT_BROWSER_PROFILE", "").strip() + if override and os.path.isabs(override): + return Path(override) + return runtime.state_dir() / "browser-profile" + + +def executable() -> Optional[str]: + """The Chromium agent-browser launches: an explicit ``AGENT_BROWSER_EXECUTABLE_PATH``, else the newest + Playwright Chromium it bundles, else a system Chrome/Chromium. ``None`` when there is none.""" + explicit = os.environ.get("AGENT_BROWSER_EXECUTABLE_PATH", "").strip() + if explicit and os.access(explicit, os.X_OK): + return explicit + from tools.browser_tool_install import _chromium_search_roots + candidates = sorted( + (p for root in _chromium_search_roots() for p in glob.glob(os.path.join(root, "chromium-*", "chrome-linux*", "chrome"))), + key=os.path.getmtime, reverse=True) + for exe in candidates: + if os.access(exe, os.X_OK): + return exe + return next((shutil.which(name) for name in _SYSTEM_BROWSERS if shutil.which(name)), None) + + +def dock_launch() -> Optional[Tuple[str, str]]: + """``(executable, user_data_dir)`` for the dock's Browser icon, or ``None`` when no Chromium exists.""" + exe = executable() + return (exe, str(profile_dir())) if exe else None + + +def env_for_agent(env: dict) -> dict: + """Pin agent-browser to the screen's browser identity unless the user pinned their own.""" + env.setdefault("AGENT_BROWSER_PROFILE", str(profile_dir())) + exe = executable() + if exe: + env.setdefault("AGENT_BROWSER_EXECUTABLE_PATH", exe) + return env diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh index 2f7348aa892b..75006e0193b6 100755 --- a/tools/bot_desktop/launcher.sh +++ b/tools/bot_desktop/launcher.sh @@ -40,8 +40,14 @@ mkdir -p "$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" "$XDG_CONFIG_HOME/a export DISPLAY=":$HERMES_BD_DISPLAY_NUM" export XAUTHORITY="$HERMES_BD_XAUTH" -# Stale lock files from a crashed server block restart. -rm -f "$HERMES_BD_SOCKET" "/tmp/.X${HERMES_BD_DISPLAY_NUM}-lock" "/tmp/.X11-unix/X${HERMES_BD_DISPLAY_NUM}" +# Stale lock files from a crashed server block restart; a lock whose pid is alive belongs to a +# running server (another profile may have taken this number) and is never touched — Xvnc then +# fails to start on it and runtime.py reports that instead of us disrupting the other desktop. +rm -f "$HERMES_BD_SOCKET" +xlock="/tmp/.X${HERMES_BD_DISPLAY_NUM}-lock" +if [[ -e "$xlock" ]] && ! kill -0 "$(tr -d ' ' < "$xlock" 2>/dev/null)" 2>/dev/null; then + rm -f "$xlock" "/tmp/.X11-unix/X${HERMES_BD_DISPLAY_NUM}" +fi : > "$XAUTHORITY"; chmod 600 "$XAUTHORITY" xauth -q -f "$XAUTHORITY" add "$DISPLAY" MIT-MAGIC-COOKIE-1 "$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" @@ -134,9 +140,9 @@ if [[ ! -e "$X/xfce4-panel.xml" ]]; then dock_ids+=("$n") } add_launcher "Terminal" utilities-terminal "xfce4-terminal" - for b in google-chrome chromium chromium-browser firefox; do - command -v "$b" >/dev/null 2>&1 && { add_launcher "Browser" internet-web-browser "$b"; break; } - done + # The bot's browser: runtime.py resolves the executable agent-browser drives plus the profile's + # persistent user-data-dir, so a human taking over lands in the bot's own cookie jar. + [[ -n "${HERMES_BD_BROWSER_EXEC:-}" ]] && add_launcher "Browser" internet-web-browser "$HERMES_BD_BROWSER_EXEC" add_launcher "Files" system-file-manager "thunar" add_launcher "Text Editor" accessories-text-editor "mousepad" dock_plugins=""; dock_items="" diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py index 405c68393650..6cc4e65c270b 100644 --- a/tools/bot_desktop/lease.py +++ b/tools/bot_desktop/lease.py @@ -5,22 +5,29 @@ credential, so even screenshots are refused; fail closed rather than trusting the agent to pause itself) and the Desktop UI (Watch / Take over / Hand back). -Per-process, keyed by ``hermes_home_key()`` so multiplexed profiles never share a lease. Handoff -requests raised by the agent (``request_handoff``) are how it asks for hands and later learns the -human is done: ``wait_for_release`` blocks the tool call until the lease returns to the agent. +Authority lives ON DISK, ``/bot-desktop/lease.json`` under an fcntl lock, because the +processes that must agree do not share memory: ``hermes serve`` (viewer bridge), the messaging +gateway, a CLI turn and isolated workers all drive the same display. Every read goes to the file; +the in-process Condition only wakes local waiters early. ``epoch`` increments on every transition so +an action admitted under one lease can tell that control changed underneath it. """ from __future__ import annotations +import fcntl +import json +import os import threading import time -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field +from pathlib import Path from typing import Callable, Dict, List, Optional -from hermes_constants import hermes_home_key +from hermes_constants import get_hermes_home, hermes_home_key AGENT = "agent" HUMAN = "human" +_POLL_SECONDS = 0.25 class HumanHasControl(RuntimeError): @@ -34,28 +41,63 @@ class Lease: since: float = field(default_factory=time.time) reason: str = "" pending_handoff: Optional[str] = None # agent's reason for asking, until the human takes over + epoch: int = 0 def as_dict(self) -> Dict[str, object]: - return {"holder": self.holder, "viewer_id": self.viewer_id, "since": self.since, - "reason": self.reason, "pending_handoff": self.pending_handoff} + return asdict(self) _lock = threading.Condition() -_leases: Dict[str, Lease] = {} _listeners: List[Callable[[str, Lease], None]] = [] -def _key(profile_key: Optional[str]) -> str: - return profile_key or hermes_home_key() +def _path(profile_key: Optional[str]) -> Path: + """``profile_key`` is the HERMES_HOME path of the profile whose lease is meant (the RFB bridge + serves several profiles from one process); ``None`` means the current profile.""" + home = Path(profile_key) if profile_key else get_hermes_home() + return home / "bot-desktop" / "lease.json" + + +def _read(path: Path) -> Lease: + try: + data = json.loads(path.read_text(encoding="utf-8")) + return Lease(**{k: v for k, v in data.items() if k in Lease.__dataclass_fields__}) + except (OSError, ValueError, TypeError): + return Lease() + + +def _write(path: Path, lease: Lease) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(lease.as_dict()), encoding="utf-8") + os.replace(tmp, path) + + +class _locked: + """Cross-process critical section over the lease file (fcntl on a sibling lock file).""" + + def __init__(self, path: Path): + self._lockfile = path.with_suffix(".lock") + self._fh = None + + def __enter__(self): + self._lockfile.parent.mkdir(parents=True, exist_ok=True) + self._fh = open(self._lockfile, "a+", encoding="utf-8") # noqa: SIM115 — closed in __exit__ + fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX) + return self + + def __exit__(self, *exc): + fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN) # type: ignore[union-attr] + self._fh.close() # type: ignore[union-attr] def get(profile_key: Optional[str] = None) -> Lease: - with _lock: - return _leases.setdefault(_key(profile_key), Lease()) + return _read(_path(profile_key)) def on_change(listener: Callable[[str, Lease], None]) -> Callable[[], None]: - """Subscribe to lease transitions (gateway broadcasts them to Desktop clients).""" + """Subscribe to lease transitions made IN THIS PROCESS (the gateway broadcasts them to Desktop + clients). Transitions made by another process are observed by reading, not by callback.""" with _lock: _listeners.append(listener) @@ -74,60 +116,66 @@ def _notify(key: str, lease: Lease) -> None: pass +def _transition(profile_key: Optional[str], mutate: Callable[[Lease], bool]) -> Lease: + key, path = hermes_home_key(profile_key) if profile_key else hermes_home_key(), _path(profile_key) + with _locked(path): + lease = _read(path) + if not mutate(lease): + return lease + lease.epoch += 1 + _write(path, lease) + with _lock: + _lock.notify_all() + _notify(key, lease) + return lease + + def acquire(viewer_id: str, *, profile_key: Optional[str] = None, reason: str = "") -> Lease: """Human ``viewer_id`` takes control. Last writer wins: a second viewer evicts the first, and the RFB bridge closes the evicted socket so its UI drops to view-only.""" - key = _key(profile_key) - with _lock: - lease = _leases.setdefault(key, Lease()) + def _m(lease: Lease) -> bool: lease.holder, lease.viewer_id, lease.since, lease.reason = HUMAN, viewer_id, time.time(), reason lease.pending_handoff = None - _lock.notify_all() - _notify(key, lease) - return lease + return True + return _transition(profile_key, _m) def release(viewer_id: Optional[str] = None, *, profile_key: Optional[str] = None) -> Lease: """Return control to the agent. With ``viewer_id`` only that holder may release (a stale viewer closing its window must not yank control from the one who took over after it).""" - key = _key(profile_key) - with _lock: - lease = _leases.setdefault(key, Lease()) + def _m(lease: Lease) -> bool: if viewer_id is not None and lease.holder == HUMAN and lease.viewer_id != viewer_id: - return lease + return False lease.holder, lease.viewer_id, lease.since, lease.reason = AGENT, None, time.time(), "" lease.pending_handoff = None # "hand back" answers an open request even if nobody formally took over - _lock.notify_all() - _notify(key, lease) - return lease + return True + return _transition(profile_key, _m) def request_handoff(reason: str, *, profile_key: Optional[str] = None) -> Lease: """Agent asks a human to take over (login, 2FA, CAPTCHA, payment). Recorded so the UI can show why and the bridge can page the user; control itself still flips only on ``acquire``.""" - key = _key(profile_key) - with _lock: - lease = _leases.setdefault(key, Lease()) + def _m(lease: Lease) -> bool: lease.pending_handoff = reason - _lock.notify_all() - _notify(key, lease) - return lease + return True + return _transition(profile_key, _m) def wait_for_release(*, timeout: float, profile_key: Optional[str] = None) -> bool: """Block until the agent holds the lease (and no handoff is pending) or ``timeout`` elapses. - True when control is back with the agent.""" - key = _key(profile_key) + True when control is back with the agent. Polls the file so a release made by another process + is seen; the local Condition just shortens the wait for same-process transitions.""" + path = _path(profile_key) deadline = time.monotonic() + timeout - with _lock: - while True: - lease = _leases.setdefault(key, Lease()) - if lease.holder == AGENT and lease.pending_handoff is None: - return True - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - _lock.wait(remaining) + while True: + lease = _read(path) + if lease.holder == AGENT and lease.pending_handoff is None: + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + with _lock: + _lock.wait(min(remaining, _POLL_SECONDS)) def human_holds(profile_key: Optional[str] = None) -> bool: @@ -139,16 +187,24 @@ def viewer_may_send_input(viewer_id: str, *, profile_key: Optional[str] = None) return lease.holder == HUMAN and lease.viewer_id == viewer_id -def assert_agent_may_act(profile_key: Optional[str] = None) -> None: +def assert_agent_may_act(profile_key: Optional[str] = None) -> Lease: + """The lease as of now, or ``HumanHasControl``. Callers keep the returned ``epoch`` and compare it + with ``get().epoch`` after an admitted action: a change means a human took over mid-flight.""" lease = get(profile_key) if lease.holder == HUMAN: raise HumanHasControl( "A human has taken over this desktop (they may be entering a credential). Screen actions and " "captures are refused until they hand control back; call computer_use action='wait_for_human' " "to block until then.") + return lease def _reset_for_tests() -> None: with _lock: - _leases.clear() _listeners.clear() + p = get_hermes_home() / "bot-desktop" / "lease.json" + for f in (p, p.with_suffix(".lock"), p.with_suffix(".json.tmp")): + try: + f.unlink() + except OSError: + pass diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py index cead59dd065d..701349ca70d6 100644 --- a/tools/bot_desktop/runtime.py +++ b/tools/bot_desktop/runtime.py @@ -119,16 +119,32 @@ def _launcher_pid() -> Optional[int]: def _display_in_use(num: int) -> bool: - return Path(f"/tmp/.X{num}-lock").exists() or Path(f"/tmp/.X11-unix/X{num}").exists() + """A live X server owns ``:num``: its lock file names a running pid. A lock left by a crashed + server (dead pid) does not count, so the number can be reclaimed.""" + lock = Path(f"/tmp/.X{num}-lock") + try: + pid = int(lock.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + return False + return _pid_alive(pid) + + +_ALLOC_LOCK = Path("/tmp/.hermes-bot-desktop-alloc.lock") # host-wide: profiles allocate from one band def _allocate_display() -> int: - recorded = _read(state_dir() / "display") - if recorded and recorded.isdigit(): - return int(recorded) - for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1): - if not _display_in_use(num): - return num + """Pick this profile's display number under a host-wide lock. The recorded number is only reused + when no OTHER server holds it now: after profile A stops, B may have taken A's old number, and + A's launcher must never unlink B's socket and lock.""" + import fcntl + with open(_ALLOC_LOCK, "a+", encoding="utf-8") as fh: # windows-footgun: ok — Linux-only runtime + fcntl.flock(fh.fileno(), fcntl.LOCK_EX) + recorded = _read(state_dir() / "display") + if recorded and recorded.isdigit() and not _display_in_use(int(recorded)): + return int(recorded) + for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1): + if not _display_in_use(num): + return num raise RuntimeError("no free X display number in the Bot Desktop band") @@ -141,11 +157,13 @@ def desktop_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: if published: env.update(published) env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends + from tools.bot_desktop.browser import env_for_agent + env_for_agent(env) # same binary + user-data-dir as the dock's Browser icon return env def ensure_started_for_tool() -> None: - """Tool-boundary hook (``computer_use`` dispatch): with ``bot_desktop.auto_start`` (default on) a Linux + """Tool-boundary hook (``computer_use`` dispatch): with ``bot_desktop.auto_start`` (opt-in, default off) a Linux host that has NO display and the packages installed gets its screen started on first use, so a headless gateway works the first time instead of answering "no DISPLAY is set". Failure is not an error here; the tool's own "no display" diagnosis is the right message then.""" @@ -164,7 +182,7 @@ def _should_auto_start(env: Dict[str, str]) -> bool: return False from hermes_cli.config import load_config_readonly cfg = load_config_readonly().get("bot_desktop") or {} - return bool(cfg.get("auto_start", True)) + return bool(cfg.get("auto_start", False)) def published_env() -> Dict[str, str]: @@ -250,6 +268,10 @@ def start(*, wait_seconds: float = 15.0) -> DesktopStatus: "HERMES_BD_CONFIG_HOME": str(sd / "xdg"), "HERMES_BD_GEOMETRY": geometry(), }) + from tools.bot_desktop.browser import dock_launch + if (browser := dock_launch()) is not None: + # first-run / default-browser dialogs would sit between the human and the bot's tabs + child_env["HERMES_BD_BROWSER_EXEC"] = f"{browser[0]} --user-data-dir={browser[1]} --no-first-run --no-default-browser-check" log = open(sd / "launcher.log", "ab") # noqa: SIM115 — handed to the child, closed by it proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host) ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, diff --git a/tools/computer_use/handoff.py b/tools/computer_use/handoff.py index 6d04f048de1f..11e30d788a7f 100644 --- a/tools/computer_use/handoff.py +++ b/tools/computer_use/handoff.py @@ -3,8 +3,9 @@ back. Both are answered without touching cua-driver, so a human typing a credential is never captured. -The notification itself (Desktop pane badge, Telegram/Discord message) is emitted by the gateway's -lease listener; this module only records intent on the lease and waits. +The Desktop pane shows the request (the gateway broadcasts the lease change). Reaching the person +anywhere else is the model's job: it relays the ask in its reply, which is what lands in the chat +surface the user is actually on. This module only records intent on the lease and waits. """ from __future__ import annotations @@ -25,8 +26,9 @@ def handle_handoff(action: str, args: Dict[str, Any]) -> str: _lease.request_handoff(reason) return json.dumps({ "ok": True, "action": action, "state": _lease.get().as_dict(), - "next": "The user has been asked to open this bot's Desktop pane and take over. Call " - "computer_use action='wait_for_human' to block until they hand control back, then " + "next": "Hermes Desktop now shows 'Bot needs you' on this bot's Screen. Tell the user in your " + "reply what to do and that they can take over from Bots > Screen, then call " + "computer_use action='wait_for_human' to block until they hand control back and " "re-capture before continuing — the screen state is whatever they left."}) timeout = min(_MAX_WAIT_SECONDS, max(1.0, float(args.get("seconds") or _DEFAULT_WAIT_SECONDS))) released = _lease.wait_for_release(timeout=timeout) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index e082c0b4c06c..c7d3c3e5050a 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -254,10 +254,12 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: # Bot Desktop lease: while a human drives the screen every action, capture included, is refused. from tools.bot_desktop import lease as _bd_lease from tools.bot_desktop.runtime import ensure_started_for_tool as _bd_ensure_started + def _refused(e: Exception) -> str: + return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)}) try: - _bd_lease.assert_agent_may_act() + admitted = _bd_lease.assert_agent_may_act() except _bd_lease.HumanHasControl as e: - return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)}) + return _refused(e) _bd_ensure_started() # headless gateway: bring the profile's screen up before the backend probes DISPLAY if (err := _reject_unsafe(action, args)) is not None: return err @@ -276,7 +278,19 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: with _backend_lock: call_lock = _backend_call_locks.setdefault(session_id, threading.RLock()) with call_lock: - return _dispatch(backend, action, args) + # Re-check under the dispatch lock: approval, backend start-up and lock waits above can take + # seconds, and a human may have taken over meanwhile. A result produced after such a flip is + # discarded too — it may picture what they typed. + try: + _bd_lease.assert_agent_may_act() + except _bd_lease.HumanHasControl as e: + return _refused(e) + result = _dispatch(backend, action, args) + if _bd_lease.get().epoch != admitted.epoch and _bd_lease.human_holds(): + return _refused(_bd_lease.HumanHasControl( + "A human took over this desktop while the action ran; its result was discarded. Call " + "computer_use action='wait_for_human' to block until they hand control back.")) + return result except Exception as e: logger.exception("computer_use %s failed", action) return json.dumps({"error": f"{action} failed: {e}"}) diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index 043b8bc90773..bf71a57c33fd 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -131,26 +131,26 @@ def _ask_password() -> str: def _line(text: str) -> None: _broadcast_global_event("display.install.log", {"profile_key": profile_key, "line": text}) - # The worker thread has no context-bound transport; the sudo card must reach the CLIENT that - # clicked Install, so the caller's transport is carried across. - from .transport import bind_transport, current_transport - caller_transport = current_transport() + # The worker thread inherits NO context: the caller's transport (so the sudo card reaches the + # CLIENT that clicked Install) and the profile scope `_profile_scoped` installed (so status, lock + # and events all speak for the requested profile) are carried across with copy_context(). + import contextvars + ctx = contextvars.copy_context() def _run() -> None: - bind_transport(caller_transport) try: code = _bd_install.install_packages(ask_password=_ask_password, on_line=_line) except Exception as e: _line(f"install failed: {e}") code = 1 _broadcast_global_event("display.install.done", {"profile_key": profile_key, "code": code, - "status": _bd_runtime.status().as_dict()}) + "status": _display_snapshot()}) try: _bd_install.assert_not_running() except _bd_install.InstallBusy as e: return _err(rid, _DISPLAY_ERR, str(e)) - threading.Thread(target=_run, name=f"bot-desktop-install:{profile_key}", daemon=True).start() + threading.Thread(target=ctx.run, args=(_run,), name=f"bot-desktop-install:{profile_key}", daemon=True).start() return _ok(rid, {"started": True, "command": _bd_runtime.install_command(), "profile_key": profile_key}) diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md index 796e0179c93a..363bc369111d 100644 --- a/website/docs/user-guide/features/bot-screen.md +++ b/website/docs/user-guide/features/bot-screen.md @@ -56,9 +56,10 @@ Every bot's computer is one click away in three places of Hermes Desktop: its conversations too. 1. Open the Screen with any of the entries above. - The first time, click **Start screen** (or leave `bot_desktop.auto_start` on, - the default: the screen starts on the bot's first `computer_use` call when the - host has no display). A headed browser opens on the screen once it is running. + The first time, click **Start screen**. Set `bot_desktop.auto_start: true` if + you want a headless host to start the screen by itself on the bot's first + `computer_use` call (off by default: installing TigerVNC never yields a screen + nobody asked for). A headed browser opens on the screen once it is running. 2. The pane streams the bot's desktop. The chip in the header says who is in control: **Bot is in control** by default. 3. Click **Take over**. The border turns red, your keyboard and mouse now drive @@ -71,18 +72,22 @@ refused with `human_has_control`; the bot never sees what you type. The bot can ask for you: when it recognises a login or verification step it calls `computer_use` with `action: "request_handoff"` and a reason, the pane -shows **Bot needs you**, and the bot blocks in `action: "wait_for_human"` until -you hand back. Your Telegram/Discord chat with the bot gets the same request. +shows **Bot needs you**, the bot tells you in its reply what it needs (so the +ask reaches you in whatever chat you are on), and it blocks in +`action: "wait_for_human"` until you hand back. Two viewers on one screen: the most recent **Take over** wins; the previous controller drops back to watching. ## Browser sessions that survive the handoff -The bot's headed Chromium (`browser.headed: true` or a real-profile session) -opens on the bot's screen and keeps one persistent profile per bot. What you -sign in to during a takeover is what the bot uses afterwards, and in every later -session for that bot, until the site itself expires the login. +While the screen runs, the bot's browser tool and the dock's **Browser** icon are +the same browser: the Chromium agent-browser drives, with one persistent +user-data-dir per bot (`/bot-desktop/browser-profile`; set +`AGENT_BROWSER_PROFILE` to pin your own). Click Browser during a takeover and you +are in the bot's own windows and cookie jar; what you sign in to is what the bot +uses afterwards and in every later session, until the site expires the login. +Set `browser.headed: true` so the bot's own browsing is visible on the screen too. ## CLI From 5f3710bc82d7e123a70e00ece70a0ad232ac82f7 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 09:22:21 -0700 Subject: [PATCH 08/55] fix(desktop): sort the clearSudoRequest import (lint) --- .../hooks/use-message-stream/gateway-event/input-requests.ts | 2 +- apps/desktop/src/components/prompt-overlays.tsx | 2 ++ apps/desktop/src/plugins/hermes-bots/screen-install.tsx | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts index ac0ab916f65e..2ad74121fb2c 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts @@ -17,9 +17,9 @@ import { $vaultCodeRequests, $vaultSaveLoginRequests, $vaultUnlockRequests, + clearSudoRequest, clearVaultCodeRequest, clearVaultSaveLoginRequest, - clearSudoRequest, clearVaultUnlockRequest, receiveApprovalRequest, setSecretRequest, diff --git a/apps/desktop/src/components/prompt-overlays.tsx b/apps/desktop/src/components/prompt-overlays.tsx index 187bd341013c..6c6c5647c2c1 100644 --- a/apps/desktop/src/components/prompt-overlays.tsx +++ b/apps/desktop/src/components/prompt-overlays.tsx @@ -80,12 +80,14 @@ function SudoDialog({ sessionId }: { sessionId: string | null }) { try { const method = request.respondMethod ?? 'sudo.respond' const reply = { password: value, request_id: request.requestId } + // Pinned to the socket the request came from: the foreground gateway may be another host. if (request.origin) { await requestGatewayForAgent<{ status?: string }>(request.origin.connectionId, request.origin.profile, method, reply) } else { await gateway.request<{ status?: string }>(method, reply) } + triggerHaptic('submit') clearSudoRequest(request.sessionId, request.requestId) } catch (error) { diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx index 11662c0df86c..2bc9a9131bab 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx @@ -45,6 +45,7 @@ export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCar setLog(prev => (prev.length >= LOG_KEEP ? [...prev.slice(1), line] : [...prev, line])) } }) + const offDone = host.onEvent('display.install.done', (event: RpcEvent) => { const payload = event.payload as { profile_key?: string; code?: number; status?: DisplayStatus } | undefined From fc39beadc35a54975b7fb3d16003de9f18195589 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:43:25 -0700 Subject: [PATCH 09/55] fix(bot-screen): browser tools obey the lease, epoch-only fence, dropped viewer link keeps exclusion --- hermes_cli/web_routers/display.py | 16 ++++- .../test_display_ws_drop_keeps_lease.py | 62 +++++++++++++++++++ tests/tools/test_bot_desktop_browser_fence.py | 61 ++++++++++++++++++ tests/tools/test_bot_desktop_lease.py | 3 +- tools/browser_tool.py | 4 +- tools/browser_tool_session.py | 30 +++++++++ tools/computer_use/tool.py | 8 ++- 7 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 tests/hermes_cli/test_display_ws_drop_keeps_lease.py create mode 100644 tests/tools/test_bot_desktop_browser_fence.py diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py index 4d81e24abba8..541be828eb7c 100644 --- a/hermes_cli/web_routers/display.py +++ b/hermes_cli/web_routers/display.py @@ -26,6 +26,7 @@ _READ_CHUNK = 64 * 1024 _CLOSE_CONTROL_TAKEN = 4000 +_CLEAN_CLOSE = frozenset({1000, 1001}) _CLOSE_DESKTOP_GONE = 4001 _CLOSE_BAD_TICKET = 4401 _CLOSE_NOT_ALLOWED = 4403 @@ -55,7 +56,11 @@ async def display_ws(ws: WebSocket) -> None: if info is None: await ws.close(code=_CLOSE_BAD_TICKET, reason="display ticket missing, expired or used") return + await _bridge(ws, info) + +async def _bridge(ws: WebSocket, info: dict) -> None: + """Pump RFB bytes between the viewer socket and THIS profile's Xvnc, gated by the lease.""" from hermes_constants import hermes_home_key from tools.bot_desktop import lease as _lease from tools.bot_desktop.rfb_filter import RfbClientFilter @@ -94,6 +99,8 @@ def _on_lease(key: str, lease) -> None: rfb_filter = RfbClientFilter(lambda: _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)) + viewer_closed = asyncio.Event() + async def rfb_to_ws() -> None: while True: chunk = await reader.read(_READ_CHUNK) @@ -105,6 +112,9 @@ async def ws_to_rfb() -> None: while True: message = await ws.receive() if message.get("type") == "websocket.disconnect": + # 1000/1001 = the viewer closed the window; anything else is a dropped link. + if message.get("code") in _CLEAN_CLOSE: + viewer_closed.set() return data = message.get("bytes") if data is None: @@ -136,8 +146,10 @@ async def watch_eviction() -> None: finally: unsubscribe() writer.close() - # Closing the viewer window hands control back; a stale holder never pins the agent out. - if _lease.viewer_may_send_input(viewer_id, profile_key=profile_home): + # Closing the viewer window hands control back. A DROPPED link (laptop lid, Wi-Fi, 1006) + # keeps the human's exclusion: they may be mid-login on that screen and the agent must not + # resume into it. The Desktop reconnects into the same lease, or the human hands back. + if viewer_closed.is_set() and _lease.viewer_may_send_input(viewer_id, profile_key=profile_home): _lease.release(viewer_id, profile_key=profile_home) try: await ws.close() diff --git a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py new file mode 100644 index 000000000000..e1ea23d7329c --- /dev/null +++ b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py @@ -0,0 +1,62 @@ +"""A dropped viewer link (1006: lid closed, Wi-Fi) must NOT hand the screen back to the agent — the +human may be mid-login on it. Only a clean close (1000/1001) releases.""" + +from __future__ import annotations + +import asyncio +import os +import tempfile + +import pytest + +from hermes_cli.web_routers import display +from tools.bot_desktop import lease + + +class _Ws: + """Just enough of a Starlette WebSocket: one disconnect message with the given close code.""" + + def __init__(self, close_code: int): + self._code = close_code + self.closed = False + + async def accept(self): + pass + + async def receive(self): + await asyncio.sleep(0.05) + return {"type": "websocket.disconnect", "code": self._code} + + async def send_bytes(self, data): + pass + + async def close(self, code=1000, reason=""): + self.closed = True + + +async def _bridge_once(close_code: int, home: str) -> lease.Lease: + sock_dir = os.path.join(home, "bot-desktop") + os.makedirs(sock_dir, exist_ok=True) + sock = os.path.join(sock_dir, "rfb.sock") + + async def _xvnc(reader, writer): # a silent framebuffer + await asyncio.sleep(1) + writer.close() + + server = await asyncio.start_unix_server(_xvnc, path=sock) + try: + info = {"hermes_home": home, "viewer_id": "desk-1"} + await display._bridge(_Ws(close_code), info) + finally: + server.close() + return lease.get(profile_key=home) + + +@pytest.mark.parametrize(("close_code", "human_keeps_control"), [(1006, True), (1000, False)]) +def test_only_a_clean_viewer_close_hands_the_screen_back(monkeypatch, close_code, human_keeps_control): + lease._reset_for_tests() + with tempfile.TemporaryDirectory() as home: + lease.acquire("desk-1", profile_key=home) + after = asyncio.run(_bridge_once(close_code, home)) + lease._reset_for_tests() + assert (after.holder == lease.HUMAN) is human_keeps_control, after diff --git a/tests/tools/test_bot_desktop_browser_fence.py b/tests/tools/test_bot_desktop_browser_fence.py new file mode 100644 index 000000000000..f791ac2419f7 --- /dev/null +++ b/tests/tools/test_bot_desktop_browser_fence.py @@ -0,0 +1,61 @@ +"""The bot's browser tools obey the screen lease: while a human holds the shared browser, nothing is +dispatched, and a command whose run crossed a takeover loses its result.""" + +from __future__ import annotations + +import json + +import pytest + +from tools.bot_desktop import lease, runtime + + +@pytest.fixture(autouse=True) +def _fresh(monkeypatch): + lease._reset_for_tests() + monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":37"}) + yield + lease._reset_for_tests() + + +def _wire(monkeypatch, commands): + from tools import browser_tool as browser + from tools import browser_tool_session as session + + monkeypatch.delenv("AGENT_BROWSER_PROFILE", raising=False) + monkeypatch.setattr(browser, "_is_camofox_mode", lambda: False) + monkeypatch.setattr(browser, "_blocked_private_page_action", lambda *a: None) + monkeypatch.setattr(session, "_browser_command_preflight", lambda: {"browser_cmd": "agent-browser"}) + monkeypatch.setattr(session, "_get_session_info", lambda *a: {"session_name": "review"}) + monkeypatch.setattr(session._cloud, "_get_browser_engine", lambda: "chrome") + monkeypatch.setattr(session._cloud, "_is_headed_mode", lambda: True) + + def spawn(*args): + commands.append(args[2]) + return {"success": True, "data": {"secret": "WHAT-THE-HUMAN-TYPED"}} + + monkeypatch.setattr(session, "_spawn_and_collect", spawn) + return browser, session + + +def test_browser_click_is_fenced_while_human_controls_shared_browser(monkeypatch): + commands: list = [] + browser, _ = _wire(monkeypatch, commands) + lease.acquire("human-viewer") + result = json.loads(browser.browser_click("e1", task_id="review")) + assert commands == [], f"human holds the lease, yet a browser command was dispatched: {commands}" + assert result.get("code") == "human_has_control" + + +def test_browser_result_crossing_a_takeover_is_discarded(monkeypatch): + commands: list = [] + browser, session = _wire(monkeypatch, commands) + + def spawn_then_takeover(*args): + lease.acquire("human-viewer") + lease.release("human-viewer") # a full cycle, control is back — the frame is still theirs + return {"success": True, "data": {"secret": "WHAT-THE-HUMAN-TYPED"}} + + monkeypatch.setattr(session, "_spawn_and_collect", spawn_then_takeover) + result = browser.browser_click("e1", task_id="review") + assert "WHAT-THE-HUMAN-TYPED" not in result diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py index 7a3665eb6f7f..aa8e7b83dce7 100644 --- a/tests/tools/test_bot_desktop_lease.py +++ b/tests/tools/test_bot_desktop_lease.py @@ -94,7 +94,8 @@ def test_takeover_during_an_admitted_action_discards_its_result(monkeypatch): monkeypatch.setattr(tool, "_get_backend", lambda session_id="": object()) def _dispatch_then_takeover(backend, action, args): - lease.acquire("human") # flips while the driver call is in flight + lease.acquire("human") # a whole take-over / hand-back cycle inside the driver call: + lease.release("human") # control is back, but the frame is still the human's turn return json.dumps({"ok": True, "action": action, "png_b64": "SECRET"}) monkeypatch.setattr(tool, "_dispatch", _dispatch_then_takeover) diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 5a8c67b3abcb..99af9a0a0708 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -819,7 +819,9 @@ def _json_with_fallback(response: Dict[str, Any], result: Dict[str, Any]) -> str def _failed_response(result: Dict[str, Any], default_error: str) -> str: - return _json_with_fallback(_err(result.get("error", default_error)), result) + # ``code`` = machine-readable refusal (human_has_control), same shape as computer_use's. + extra = {"code": result["code"]} if result.get("code") else {} + return _json_with_fallback(_err(result.get("error", default_error), **extra), result) def _tool_response(result: Dict[str, Any], ok: Dict[str, Any], default_error: str) -> str: diff --git a/tools/browser_tool_session.py b/tools/browser_tool_session.py index 4b9b353b3e03..14fa7edceda6 100644 --- a/tools/browser_tool_session.py +++ b/tools/browser_tool_session.py @@ -574,6 +574,36 @@ def _run_browser_command( except Exception as e: _bt.logger.warning("Failed to create browser session for task=%s: %s", task_id, e) return {"success": False, "error": f"Failed to create browser session: {str(e)}"} + # The bot's LOCAL browser lives on its Bot Desktop screen, in the same profile a human who took + # over is typing into. While the human holds the lease every action AND read against it is + # refused (the page may show their credential); the fence brackets the whole run so a takeover + # mid-command also voids the result. Cloud / user-supplied CDP sessions are a different browser. + if _shares_bot_desktop_browser(session_info): + from tools.bot_desktop import lease as _bd_lease + try: + admitted = _bd_lease.assert_agent_may_act() + except _bd_lease.HumanHasControl as e: + return {"success": False, "error": str(e), "code": "human_has_control"} + result = _run_browser_command_unfenced(task_id, command, args, timeout, _engine_override, browser_cmd, session_info) + if _bd_lease.get().epoch != admitted.epoch: + return {"success": False, "code": "human_has_control", + "error": "A human took over the bot's screen while this browser command ran; its result was " + "discarded. Call computer_use action='wait_for_human' to block until they hand back."} + return result + return _run_browser_command_unfenced(task_id, command, args, timeout, _engine_override, browser_cmd, session_info) + + +def _shares_bot_desktop_browser(session_info: Dict[str, Any]) -> bool: + """Local agent-browser session (no CDP url: the dispatcher runs it with ``--session``, i.e. on this + host's display) while this profile's Bot Desktop screen is running.""" + if session_info.get("cdp_url"): + return False + from tools.bot_desktop import runtime as _bd_runtime + return bool(_bd_runtime.published_env().get("DISPLAY")) + + +def _run_browser_command_unfenced(task_id: str, command: str, args: List[str], timeout: int, + _engine_override: Optional[str], browser_cmd, session_info: Dict[str, Any]) -> Dict[str, Any]: # Cleanup stops the supervisor before closing the backend; keep it stopped. if command != "close" and session_info.get("cdp_url"): _cdp._ensure_cdp_supervisor(task_id) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index c7d3c3e5050a..4476de4b49ca 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -286,10 +286,12 @@ def _refused(e: Exception) -> str: except _bd_lease.HumanHasControl as e: return _refused(e) result = _dispatch(backend, action, args) - if _bd_lease.get().epoch != admitted.epoch and _bd_lease.human_holds(): + # Any lease transition during the run voids the result — including a full take-over / + # hand-back cycle that already finished: the frame still belongs to the human's turn. + if _bd_lease.get().epoch != admitted.epoch: return _refused(_bd_lease.HumanHasControl( - "A human took over this desktop while the action ran; its result was discarded. Call " - "computer_use action='wait_for_human' to block until they hand control back.")) + "A human took over this desktop while the action ran; its result was discarded. " + "Re-capture (or call computer_use action='wait_for_human' if they still hold control).")) return result except Exception as e: logger.exception("computer_use %s failed", action) From dcba5354d16277e6bd76a981ea28c2c79996da59 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:50:50 -0700 Subject: [PATCH 10/55] fix(bot-screen): events pinned to the bot's connection, stale hero frames marked, lease fails closed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second review round (@Julientalbot): - Desktop `display.*` listeners (lease, install log/done) match on (connectionId, profile_key), not the profile path alone: two hosts with the same ~/.hermes path no longer repaint each other's screen pane or install card. One predicate, `isEventForBotScreen`. - Hero thumbnail: three consecutive failed refreshes dim the last frame and caption it "Last seen — screen unreachable"; a dead gateway never keeps looking live. - Lease file present but unparsable reads as HUMAN holds (fail closed); only a missing file is a fresh agent-held profile. The next successful write repairs it. - Taking over after `request_handoff` keeps the agent's reason on the lease and the pane shows it while the human acts, not only before they clicked Take over. --- apps/desktop/src/plugins/hermes-bots/i18n.ts | 5 +++ .../hermes-bots/screen-connection.test.ts | 44 +++++++++++++++++++ .../plugins/hermes-bots/screen-connection.ts | 20 ++++++++- .../src/plugins/hermes-bots/screen-hero.tsx | 20 ++++++--- .../plugins/hermes-bots/screen-install.tsx | 12 ++--- .../src/plugins/hermes-bots/screen-pane.tsx | 11 +++-- tests/tools/test_bot_desktop_lease.py | 21 +++++++++ tools/bot_desktop/lease.py | 20 +++++++-- 8 files changed, 133 insertions(+), 20 deletions(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index 5eef7a47735d..637a9ee4b9fb 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -242,6 +242,7 @@ type BotsMessages = { heroStopped: string heroNotInstalled: string heroConnecting: string + heroStale: string heroOpenLive: string heroInstall: string heroStart: string @@ -508,6 +509,7 @@ const en: BotsMessages = { heroStopped: 'Screen is off', heroNotInstalled: 'Not installed on this host', heroConnecting: 'Checking the screen…', + heroStale: 'Last seen — screen unreachable', heroOpenLive: 'Open live', heroInstall: 'Install', heroStart: 'Start', @@ -769,6 +771,7 @@ const ja: BotsMessages = { heroStopped: '画面は停止中', heroNotInstalled: 'このホストには未インストール', heroConnecting: '画面を確認中…', + heroStale: '最終表示 — 画面に接続できません', heroOpenLive: 'ライブで開く', heroInstall: 'インストール', heroStart: '開始', @@ -1025,6 +1028,7 @@ const zh: BotsMessages = { heroStopped: '屏幕已关闭', heroNotInstalled: '此主机未安装', heroConnecting: '正在检查屏幕…', + heroStale: '最后画面 — 屏幕无法访问', heroOpenLive: '实时打开', heroInstall: '安装', heroStart: '启动', @@ -1281,6 +1285,7 @@ const zhHant: BotsMessages = { heroStopped: '螢幕已關閉', heroNotInstalled: '此主機未安裝', heroConnecting: '正在檢查螢幕…', + heroStale: '最後畫面 — 螢幕無法連線', heroOpenLive: '即時開啟', heroInstall: '安裝', heroStart: '啟動', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts new file mode 100644 index 000000000000..f4f36759bf73 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts @@ -0,0 +1,44 @@ +/** + * Two remote hosts can share the same `~/.hermes` path, so a `display.*` event + * matched on `profile_key` alone would let host B's take-over repaint host A's + * screen pane. The event must also have arrived on the bot's own connection. + */ + +import { describe, expect, it, vi } from 'vitest' + +import type { RosterRow } from './types' + +const routeMock = vi.fn<() => { connectionId: string; profile: string } | null>(() => null) + +vi.mock('@hermes/plugin-sdk', () => ({ + host: { requestProfile: vi.fn() }, + resolveSiblingWsUrl: vi.fn() +})) + +vi.mock('./routing', () => ({ + botConnectionRoute: () => routeMock() +})) + +import { isEventForBotScreen } from './screen-connection' + +const bot = { name: 'ops' } as RosterRow +const key = '/home/hermes/.hermes' + +describe('isEventForBotScreen', () => { + it('ignores a same-profile-path event that arrived from another host', () => { + routeMock.mockReturnValue({ connectionId: 'conn-a', profile: 'ops' }) + + const fromB = { connectionId: 'conn-b', payload: { profile_key: key }, type: 'display.lease' } + const fromA = { connectionId: 'conn-a', payload: { profile_key: key }, type: 'display.lease' } + + expect(isEventForBotScreen(bot, fromB, key)).toBe(false) + expect(isEventForBotScreen(bot, fromA, key)).toBe(true) + }) + + it('still matches the untagged local socket for a local bot', () => { + routeMock.mockReturnValue({ connectionId: 'local', profile: 'ops' }) + + expect(isEventForBotScreen(bot, { payload: { profile_key: key }, type: 'display.lease' }, key)).toBe(true) + expect(isEventForBotScreen(bot, { payload: { profile_key: '/other' }, type: 'display.lease' }, key)).toBe(false) + }) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts index 247a79ec0e04..75daaef936b1 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -11,7 +11,7 @@ */ import { host, resolveSiblingWsUrl } from '@hermes/plugin-sdk' -import type { PluginProfileRoute } from '@hermes/plugin-sdk' +import type { PluginProfileRoute, RpcEvent } from '@hermes/plugin-sdk' import { botConnectionRoute } from './routing' import type { RosterRow } from './types' @@ -54,6 +54,24 @@ export function botScreenRoute(bot: RosterRow): PluginProfileRoute | string { return botConnectionRoute(bot) ?? bot.name } +/** + * Does a `display.*` event belong to `bot`'s screen? Two hosts can share the same + * `~/.hermes` path, so the profile key alone is ambiguous: the event must also have + * arrived on the bot's registry connection (local/legacy events carry no tag). + */ +export function isEventForBotScreen(bot: RosterRow, event: RpcEvent, profileKey: null | string | undefined): boolean { + const payload = event.payload as { profile_key?: string } | undefined + + if (!profileKey || payload?.profile_key !== profileKey) { + return false + } + + const expected = botConnectionRoute(bot)?.connectionId ?? null + const actual = event.connectionId ?? null + + return expected === actual || (expected === 'local' && actual === null) +} + export function displayRequest(bot: RosterRow, method: string, params: Record = {}): Promise { return host.requestProfile(botScreenRoute(bot), method, params) } diff --git a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx index 1154668df7f2..32bca2d1fd6e 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx @@ -17,14 +17,19 @@ import { type PortalTone, useScreenPortalState } from './screen-portal' import type { BotMeta, RosterRow } from './types' const REFRESH_MS = 4000 +const STALE_AFTER = 3 function useLiveThumbnail(bot: RosterRow, running: boolean) { const [dataUrl, setDataUrl] = useState(null) + // Consecutive failed refreshes; past STALE_AFTER the frame is shown dimmed as "last seen" so a + // dead gateway never keeps looking live. Success resets it. + const [misses, setMisses] = useState(0) const boxRef = useRef(null) useEffect(() => { if (!running) { setDataUrl(null) + setMisses(0) return } @@ -59,10 +64,13 @@ function useLiveThumbnail(bot: RosterRow, running: boolean) { .then(result => { if (!cancelled) { setDataUrl(result.data_url) + setMisses(0) } }) .catch(() => { - /* transient: the next tick retries; the last good frame stays up */ + if (!cancelled) { + setMisses(prev => prev + 1) // the last good frame stays up, marked stale past STALE_AFTER + } }) .finally(() => { if (!cancelled) { @@ -83,7 +91,7 @@ function useLiveThumbnail(bot: RosterRow, running: boolean) { } }, [bot, running]) - return { dataUrl, boxRef } + return { dataUrl, boxRef, stale: misses >= STALE_AFTER } } const TONE_RING: Partial> = { @@ -95,13 +103,13 @@ export function ScreenHero({ bot, meta }: { bot: RosterRow; meta?: BotMeta | nul const t = useBots() const { tone } = useScreenPortalState(bot) const running = tone === 'live' || tone === 'human' || tone === 'other' - const { dataUrl, boxRef } = useLiveThumbnail(bot, running) + const { dataUrl, boxRef, stale } = useLiveThumbnail(bot, running) if (tone === 'unsupported') { return null } - const caption = { + const caption = stale ? t.screen.heroStale : { live: t.screen.portalWatching, human: t.screen.portalYouControl, other: t.screen.portalOtherControls, @@ -123,7 +131,7 @@ export function ScreenHero({ bot, meta }: { bot: RosterRow; meta?: BotMeta | nul type="button" > {dataUrl ? ( - + ) : ( @@ -131,7 +139,7 @@ export function ScreenHero({ bot, meta }: { bot: RosterRow; meta?: BotMeta | nul )} - + {t.screen.portalTitle} {caption} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx index 2bc9a9131bab..d69498ba5a3c 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx @@ -14,7 +14,7 @@ import type { RpcEvent } from '@hermes/plugin-sdk' import { useCallback, useEffect, useRef, useState } from 'react' import { useBots } from './i18n' -import { displayRequest, type DisplayStatus } from './screen-connection' +import { displayRequest, type DisplayStatus, isEventForBotScreen } from './screen-connection' import type { RosterRow } from './types' const LOG_KEEP = 200 @@ -38,18 +38,18 @@ export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCar useEffect(() => { const offLog = host.onEvent('display.install.log', (event: RpcEvent) => { - const payload = event.payload as { profile_key?: string; line?: string } | undefined + const payload = event.payload as { line?: string } | undefined - if (payload?.profile_key === status.profile_key && typeof payload.line === 'string') { + if (isEventForBotScreen(bot, event, status.profile_key) && typeof payload?.line === 'string') { const line = payload.line setLog(prev => (prev.length >= LOG_KEEP ? [...prev.slice(1), line] : [...prev, line])) } }) const offDone = host.onEvent('display.install.done', (event: RpcEvent) => { - const payload = event.payload as { profile_key?: string; code?: number; status?: DisplayStatus } | undefined + const payload = event.payload as { code?: number; status?: DisplayStatus } | undefined - if (payload?.profile_key !== status.profile_key) { + if (!payload || !isEventForBotScreen(bot, event, status.profile_key)) { return } @@ -66,7 +66,7 @@ export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCar offLog() offDone() } - }, [onInstalled, status.profile_key, t.screen.installCancelled, t.screen.installFailed]) + }, [bot, onInstalled, status.profile_key, t.screen.installCancelled, t.screen.installFailed]) const install = useCallback(async () => { setPhase('running') diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index e4cf6a12e9b9..bfe65b6d69a4 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -15,7 +15,7 @@ import type { RpcEvent } from '@hermes/plugin-sdk' import { useCallback, useEffect, useRef, useState } from 'react' import { useBots } from './i18n' -import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, resolveScreenWsUrl, VIEWER_ID } from './screen-connection' +import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isEventForBotScreen, resolveScreenWsUrl, VIEWER_ID } from './screen-connection' import { ScreenInstallCard } from './screen-install' import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' import type { RosterRow } from './types' @@ -70,9 +70,9 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { void refresh() return host.onEvent('display.lease', (event: RpcEvent) => { - const payload = event.payload as { profile_key?: string; lease?: DisplayLease } | undefined + const payload = event.payload as { lease?: DisplayLease } | undefined - if (payload?.lease && payload.profile_key && payload.profile_key === status?.profile_key) { + if (payload?.lease && isEventForBotScreen(bot, event, status?.profile_key)) { setScreenLease(bot, payload.lease) } }) @@ -257,6 +257,11 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { {t.screen.handoffRequested} + ) : lease?.holder === 'human' && lease.reason ? ( + // The agent's ask stays readable WHILE the human acts, not only before Take over. + + {lease.reason} + ) : null} {iHold ? ( {t.screen.youControl} diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py index aa8e7b83dce7..3b9da1a2cfe7 100644 --- a/tests/tools/test_bot_desktop_lease.py +++ b/tests/tools/test_bot_desktop_lease.py @@ -101,3 +101,24 @@ def _dispatch_then_takeover(backend, action, args): monkeypatch.setattr(tool, "_dispatch", _dispatch_then_takeover) res = json.loads(tool.handle_computer_use({"action": "capture"})) assert res["code"] == "human_has_control" and "SECRET" not in json.dumps(res) + + +def test_unreadable_lease_file_fails_closed_and_takeover_keeps_the_agents_reason(tmp_path): + """Missing file = fresh profile (agent). A file that exists but cannot be parsed must not read as + "agent holds": a torn write must never let the agent act on a human's screen. Taking over after a + request keeps the agent's reason so the human still sees WHY while they act.""" + from hermes_constants import hermes_home_key + + home = str(tmp_path) + assert lease.get(profile_key=home).holder == lease.AGENT + path = lease._path(home) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{ torn", encoding="utf-8") + assert lease.get(profile_key=home).holder == lease.HUMAN + lease.release(profile_key=home) # a successful write repairs it + assert lease.get(profile_key=home).holder == lease.AGENT + + lease.request_handoff("log in to the bank, 2FA on your phone", profile_key=home) + held = lease.acquire("desk-1", profile_key=home) + assert held.pending_handoff is None and held.reason == "log in to the bank, 2FA on your phone" + assert hermes_home_key(home) # sanity: the key derivation used by the bridge is available diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py index 6cc4e65c270b..2a3b84d5e5d3 100644 --- a/tools/bot_desktop/lease.py +++ b/tools/bot_desktop/lease.py @@ -59,11 +59,20 @@ def _path(profile_key: Optional[str]) -> Path: def _read(path: Path) -> Lease: + """No file = fresh profile, agent holds. A file that exists but cannot be parsed is a torn write + or tampering: fail CLOSED (human holds) — an unreadable lease must never let the agent act on a + screen a human may be using; the next successful write repairs it.""" try: - data = json.loads(path.read_text(encoding="utf-8")) - return Lease(**{k: v for k, v in data.items() if k in Lease.__dataclass_fields__}) - except (OSError, ValueError, TypeError): + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: return Lease() + except OSError: + return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file unreadable") + try: + data = json.loads(raw) + return Lease(**{k: v for k, v in data.items() if k in Lease.__dataclass_fields__}) + except (ValueError, TypeError): + return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file corrupt") def _write(path: Path, lease: Lease) -> None: @@ -134,7 +143,10 @@ def acquire(viewer_id: str, *, profile_key: Optional[str] = None, reason: str = """Human ``viewer_id`` takes control. Last writer wins: a second viewer evicts the first, and the RFB bridge closes the evicted socket so its UI drops to view-only.""" def _m(lease: Lease) -> bool: - lease.holder, lease.viewer_id, lease.since, lease.reason = HUMAN, viewer_id, time.time(), reason + # The agent's ask ("please log in to X") stays as the takeover reason: the human needs it + # on screen WHILE they act, not only before they clicked Take over. + lease.holder, lease.viewer_id, lease.since = HUMAN, viewer_id, time.time() + lease.reason = reason or lease.pending_handoff or "" lease.pending_handoff = None return True return _transition(profile_key, _m) From f89e71e05126fb792723db7e506c975657a0e423 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:02:52 -0700 Subject: [PATCH 11/55] fix(bot-desktop): lease no longer needs fcntl at import time computer_use imports tools.computer_use.handoff (and the lease) on every non-empty action, so the module-level `import fcntl` made every desktop action raise ModuleNotFoundError on native Windows / fcntl-less hosts. The import is now optional; the lease file semantics are unchanged and only the cross-process lock degrades to a no-op where no multi-process Bot Desktop exists. (cherry picked from commit 82ca07dda5d73741a5d6ac60f7b596dd156ec20c) --- tests/tools/test_bot_desktop_lease.py | 21 +++++++++++++++++++++ tools/bot_desktop/lease.py | 14 +++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py index 3b9da1a2cfe7..e67d664af635 100644 --- a/tests/tools/test_bot_desktop_lease.py +++ b/tests/tools/test_bot_desktop_lease.py @@ -122,3 +122,24 @@ def test_unreadable_lease_file_fails_closed_and_takeover_keeps_the_agents_reason held = lease.acquire("desk-1", profile_key=home) assert held.pending_handoff is None and held.reason == "log in to the bank, 2FA on your phone" assert hermes_home_key(home) # sanity: the key derivation used by the bridge is available + + +def test_lease_works_without_fcntl(tmp_path): + """Windows and fcntl-less hosts: ``computer_use`` imports the lease (via handoff) on EVERY call, so a + module-level fcntl dependency turns every desktop action into ModuleNotFoundError there. The file + semantics must still work; only the cross-process lock degrades. Subprocess so the module cache is clean.""" + import os + import subprocess + import sys + + probe = ("import sys; sys.modules['fcntl'] = None; sys.path.insert(0, %r)\n" + "from tools.bot_desktop import lease\n" + "import tools.computer_use.handoff\n" + "assert lease.get().holder == lease.AGENT\n" + "assert lease.acquire('v1').holder == lease.HUMAN\n" + "assert lease.get().holder == lease.HUMAN\n" + "assert lease.release('v1').holder == lease.AGENT\n" + "print('OK')\n") % os.getcwd() + out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, encoding="utf-8", timeout=60, + stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": str(tmp_path)}) + assert out.stdout.strip() == "OK", out.stderr diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py index 2a3b84d5e5d3..f890677dc386 100644 --- a/tools/bot_desktop/lease.py +++ b/tools/bot_desktop/lease.py @@ -14,7 +14,6 @@ from __future__ import annotations -import fcntl import json import os import threading @@ -25,6 +24,11 @@ from hermes_constants import get_hermes_home, hermes_home_key +try: + import fcntl +except ImportError: # Windows/macOS without fcntl: computer_use imports this module on every call, and no + fcntl = None # multi-process Bot Desktop exists there, so the cross-process lock degrades to a no-op. + AGENT = "agent" HUMAN = "human" _POLL_SECONDS = 0.25 @@ -90,14 +94,18 @@ def __init__(self, path: Path): self._fh = None def __enter__(self): + if fcntl is None: + return self self._lockfile.parent.mkdir(parents=True, exist_ok=True) self._fh = open(self._lockfile, "a+", encoding="utf-8") # noqa: SIM115 — closed in __exit__ fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX) return self def __exit__(self, *exc): - fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN) # type: ignore[union-attr] - self._fh.close() # type: ignore[union-attr] + if self._fh is None: + return + fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN) + self._fh.close() def get(profile_key: Optional[str] = None) -> Lease: From 36c0489342ef3ca33ea2e58cd898f46bb5057121 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:02:52 -0700 Subject: [PATCH 12/55] chore(windows-footguns): flag module-level imports of POSIX-only stdlib modules fcntl/pwd/grp/termios/resource/pty/tty fail at import time on Windows and take every importer down with them; the lease regression slipped through because no rule covered this class. Honours the existing `# windows-footgun: ok` marker; scoped to unindented imports so lazy and try/except ImportError forms pass. (cherry picked from commit aa625e7d2649dbe4e4357da712e492cb97f66acc) --- scripts/check-windows-footguns.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/check-windows-footguns.py b/scripts/check-windows-footguns.py index 30100402d0ad..d5de92b410c1 100644 --- a/scripts/check-windows-footguns.py +++ b/scripts/check-windows-footguns.py @@ -431,6 +431,27 @@ class Footgun: and _call_closes_on_line(line, m.end()) ), ), + Footgun( + name="module-level import of a POSIX-only stdlib module", + # Only unindented imports: a top-level `import fcntl` fails at import time on Windows and takes + # every importer down with it (tools.bot_desktop.lease took computer_use down on native + # Windows). Indented imports inside a function or a try/except ImportError are the fix shape. + pattern=re.compile( + r"^(?:import\s+(?:fcntl|pwd|grp|termios|resource|pty|tty)\b" + r"|from\s+(?:fcntl|pwd|grp|termios|resource|pty|tty)\s+import\b)" + ), + message=( + "fcntl/pwd/grp/termios/resource/pty/tty do not exist on Windows; a module-level import " + "raises ModuleNotFoundError and breaks every module that imports this one." + ), + fix=( + "Import lazily inside the function that needs it, or\n" + "try:\n" + " import fcntl\n" + "except ImportError:\n" + " fcntl = None # and take the Windows path when None" + ), + ), ] From dbd5d608b6de5ccb072af781140f4e27581d0e57 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:02:52 -0700 Subject: [PATCH 13/55] fix(display): mark the lease listener installed only after it is subscribed The Event was set before the import and on_change() ran, so a failure there left the flag set and no listener ever installed: every later lease transition would go unbroadcast for the life of the process. (cherry picked from commit c5d43c620fc9ea1428fa546b079ceeca6dbc7ef3) --- tui_gateway/methods_display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index bf71a57c33fd..772668916974 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -38,12 +38,12 @@ def _install_lease_listener() -> None: """Once per process: broadcast every lease change to all connected clients.""" if _lease_listener_installed.is_set(): return - _lease_listener_installed.set() from tools.bot_desktop import lease as _bd_lease def _on_change(profile_key: str, lease) -> None: _broadcast_global_event("display.lease", {"profile_key": profile_key, "lease": lease.as_dict()}) _bd_lease.on_change(_on_change) + _lease_listener_installed.set() # only once the subscription exists, or a failed import would silence every client @method("display.status") From ee3507bbf13bce9308c8876c75d254f02eadf156 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:04:07 -0700 Subject: [PATCH 14/55] fix(browser): fence the bot's browser by provenance, not by cdp_url Real-profile local sessions attach over a loopback cdp_url, yet that Chrome is launched with the Bot Desktop DISPLAY, so it is the very browser a human who took over is typing into; keying the fence on "no cdp_url" let every command through. Fence whenever the session carries the `local` feature and exempt only remote/cloud/user-supplied CDP. Also fence while a human holds the lease even when the published DISPLAY is gone (dead Xvnc), matching computer_use instead of silently unfencing. (cherry picked from commit 7b8825bf32a67229666f1f848d3ac171ff3fbc93) --- tests/tools/test_bot_desktop_browser_fence.py | 19 ++++++++++++++++++- tools/browser_tool_session.py | 12 +++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/tools/test_bot_desktop_browser_fence.py b/tests/tools/test_bot_desktop_browser_fence.py index f791ac2419f7..743deec2de8b 100644 --- a/tests/tools/test_bot_desktop_browser_fence.py +++ b/tests/tools/test_bot_desktop_browser_fence.py @@ -26,7 +26,7 @@ def _wire(monkeypatch, commands): monkeypatch.setattr(browser, "_is_camofox_mode", lambda: False) monkeypatch.setattr(browser, "_blocked_private_page_action", lambda *a: None) monkeypatch.setattr(session, "_browser_command_preflight", lambda: {"browser_cmd": "agent-browser"}) - monkeypatch.setattr(session, "_get_session_info", lambda *a: {"session_name": "review"}) + monkeypatch.setattr(session, "_get_session_info", lambda *a: {"session_name": "review", "cdp_url": None, "features": {"local": True}}) monkeypatch.setattr(session._cloud, "_get_browser_engine", lambda: "chrome") monkeypatch.setattr(session._cloud, "_is_headed_mode", lambda: True) @@ -59,3 +59,20 @@ def spawn_then_takeover(*args): monkeypatch.setattr(session, "_spawn_and_collect", spawn_then_takeover) result = browser.browser_click("e1", task_id="review") assert "WHAT-THE-HUMAN-TYPED" not in result + + +def test_real_profile_local_browser_is_fenced_by_provenance_even_without_a_live_display(monkeypatch): + """A real-profile session attaches over a loopback cdp_url but is launched with the Bot Desktop + DISPLAY, so it IS the human's browser: the fence keys on the ``local`` feature, not on the + transport. And a stranded human lease with the screen already down must still fence (computer_use + does), not silently unfence the browser.""" + commands: list = [] + browser, session = _wire(monkeypatch, commands) + monkeypatch.setattr(session, "_get_session_info", lambda *a: { + "session_name": "rp_1", "cdp_url": "ws://127.0.0.1:9222/devtools/browser/x", + "features": {"local": True, "real_profile": True}}) + monkeypatch.setattr(runtime, "published_env", lambda: {}) + lease.acquire("human-viewer") + result = json.loads(browser.browser_click("e1", task_id="review")) + assert commands == [], f"human holds the lease, yet a real-profile browser command was dispatched: {commands}" + assert result.get("code") == "human_has_control" diff --git a/tools/browser_tool_session.py b/tools/browser_tool_session.py index 14fa7edceda6..574491592349 100644 --- a/tools/browser_tool_session.py +++ b/tools/browser_tool_session.py @@ -594,12 +594,14 @@ def _run_browser_command( def _shares_bot_desktop_browser(session_info: Dict[str, Any]) -> bool: - """Local agent-browser session (no CDP url: the dispatcher runs it with ``--session``, i.e. on this - host's display) while this profile's Bot Desktop screen is running.""" - if session_info.get("cdp_url"): + """Decided by provenance, not transport: every LOCAL session (plain ``--session``, real-profile CDP + attach, Lightpanda) is a browser Hermes launched with this profile's Bot Desktop DISPLAY, so it is the + screen a human who took over is typing into. Cloud / user-supplied CDP sessions are another browser. + A human lease with the screen already gone (dead Xvnc) still fences — computer_use does the same.""" + if not (session_info.get("features") or {}).get("local"): return False - from tools.bot_desktop import runtime as _bd_runtime - return bool(_bd_runtime.published_env().get("DISPLAY")) + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + return bool(_bd_runtime.published_env().get("DISPLAY")) or _bd_lease.human_holds() def _run_browser_command_unfenced(task_id: str, command: str, args: List[str], timeout: int, From b68f67dc67bf6bd061d72fe07cb2ddd645a85e57 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:06:11 -0700 Subject: [PATCH 15/55] fix(computer-use): fence a captured frame before it is persisted or sent to aux vision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The epoch check ran only after _dispatch() returned, by which point the capture path had already written the PNG to the media cache, spilled the element tree to disk and routed the frame through auxiliary vision — the human's screen had left the process before being "discarded". A fence callable is threaded into _dispatch; capture and capture_after call it the moment backend.capture() returns, and the post-dispatch check stays for every other action. (cherry picked from commit 70c09e5fad89d2817f13aea772bd845ab7cc4c8a) --- tests/tools/test_bot_desktop_lease.py | 2 +- .../tools/test_computer_use_capture_fence.py | 56 +++++++++++++++++++ tools/computer_use/tool.py | 44 +++++++++------ 3 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 tests/tools/test_computer_use_capture_fence.py diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py index e67d664af635..f1ada766cef1 100644 --- a/tests/tools/test_bot_desktop_lease.py +++ b/tests/tools/test_bot_desktop_lease.py @@ -93,7 +93,7 @@ def test_takeover_during_an_admitted_action_discards_its_result(monkeypatch): monkeypatch.setattr(tool, "_get_backend", lambda session_id="": object()) - def _dispatch_then_takeover(backend, action, args): + def _dispatch_then_takeover(backend, action, args, **_): lease.acquire("human") # a whole take-over / hand-back cycle inside the driver call: lease.release("human") # control is back, but the frame is still the human's turn return json.dumps({"ok": True, "action": action, "png_b64": "SECRET"}) diff --git a/tests/tools/test_computer_use_capture_fence.py b/tests/tools/test_computer_use_capture_fence.py new file mode 100644 index 000000000000..65736e5ca12e --- /dev/null +++ b/tests/tools/test_computer_use_capture_fence.py @@ -0,0 +1,56 @@ +"""A frame captured across a human takeover never leaves the process: the lease fence runs as soon as the +backend hands the frame back, before it is persisted to the media cache, spilled, or routed to aux vision.""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +from tools.bot_desktop import lease +from tools.computer_use.backend import ActionResult, CaptureResult + + +@pytest.fixture(autouse=True) +def _fresh_lease(): + lease._reset_for_tests() + yield + lease._reset_for_tests() + + +class _TakeoverBackend: + """Driver whose capture returns while a take-over / hand-back cycle happened underneath it.""" + _last_target = None + _last_app = None + + def capture(self, **kw): + lease.acquire("human") + lease.release("human") + return CaptureResult(mode="som", width=64, height=64, png_b64=base64.b64encode(b"SECRETPNG").decode(), elements=[], + app="Bank", window_title="login") + + def click(self, **kw): + return ActionResult(ok=True, action="click") + + +def _spy_sinks(monkeypatch, tool): + leaked: list = [] + monkeypatch.setattr(tool, "_persist_capture_image", lambda cap: leaked.append(("persist", cap))) + monkeypatch.setattr(tool, "_spill_elements_to_file", lambda cap: leaked.append(("spill", cap))) + monkeypatch.setattr(tool, "_should_route_through_aux_vision", lambda: leaked.append(("aux-decide",)) or True) + monkeypatch.setattr(tool, "_route_capture_through_aux_vision", lambda cap, summary, **kw: leaked.append(("aux", cap))) + return leaked + + +@pytest.mark.parametrize("args", [{"action": "capture"}, {"action": "click", "coordinate": [1, 1], "capture_after": True}]) +def test_frame_captured_across_a_takeover_is_dropped_before_any_sink(monkeypatch, args): + from tools.computer_use import tool + + monkeypatch.setattr(tool, "_get_backend", lambda session_id="": _TakeoverBackend()) + monkeypatch.setattr(tool, "_request_approval", lambda *a, **k: None) + leaked = _spy_sinks(monkeypatch, tool) + res = json.loads(tool.handle_computer_use(args)) + assert res["code"] == "human_has_control", res + assert leaked == [], f"the human's frame reached a sink: {leaked}" + assert "SECRETPNG" not in json.dumps(res) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 4476de4b49ca..04b6715795c4 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -285,14 +285,20 @@ def _refused(e: Exception) -> str: _bd_lease.assert_agent_may_act() except _bd_lease.HumanHasControl as e: return _refused(e) - result = _dispatch(backend, action, args) - # Any lease transition during the run voids the result — including a full take-over / - # hand-back cycle that already finished: the frame still belongs to the human's turn. - if _bd_lease.get().epoch != admitted.epoch: - return _refused(_bd_lease.HumanHasControl( - "A human took over this desktop while the action ran; its result was discarded. " - "Re-capture (or call computer_use action='wait_for_human' if they still hold control).")) + + def _fence() -> None: + # Any lease transition since admission voids the frame — including a full take-over / + # hand-back cycle that already finished: it still belongs to the human's turn. Capture + # paths call this BEFORE the frame is persisted, spilled or sent to auxiliary vision. + if _bd_lease.get().epoch != admitted.epoch: + raise _bd_lease.HumanHasControl( + "A human took over this desktop while the action ran; its result was discarded. " + "Re-capture (or call computer_use action='wait_for_human' if they still hold control).") + result = _dispatch(backend, action, args, fence=_fence) + _fence() return result + except _bd_lease.HumanHasControl as e: + return _refused(e) except Exception as e: logger.exception("computer_use %s failed", action) return json.dumps({"error": f"{action} failed: {e}"}) @@ -361,12 +367,13 @@ def _do_scroll(backend, action, args, **delivery): return backend.scroll(direction=args.get("direction", "down"), amount=int(args.get("amount", 3)), element=args.get("element"), **_scroll_xy(args), modifiers=args.get("modifiers"), **delivery) -def _do_capture(backend, action, args, **_): +def _do_capture(backend, action, args, fence=lambda: None, **_): if (mode := str(args.get("mode", "som"))) not in {"som", "vision", "ax"}: return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"}) # pid/window_id forwarded only when given so older backends keep their defaults. - return _capture_response(backend.capture(mode=mode, app=args.get("app"), - **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None})) + cap = backend.capture(mode=mode, app=args.get("app"), **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None}) + fence() + return _capture_response(cap) def _do_listing(backend, action, args, key, **_): return json.dumps({key: (items := getattr(backend, action)()), "count": len(items)}) @@ -416,7 +423,9 @@ def _summarize_click(action: str, args: Dict[str, Any], fg: str) -> str: "input_text": "type", "screenshot": "capture", "get_window_state": "capture", "left_click": "click", "mouse_click": "click", } -def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> Any: +def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any], fence: Callable[[], None] = lambda: None) -> Any: + """``fence`` raises when the screen lease moved since admission; capture paths call it as soon as the + frame is in hand, before anything derived from it leaves the process.""" spec = _ACTIONS.get(action) if spec is None: return json.dumps({"error": f"unknown action {action!r}" + (f" — did you mean {hint!r}? See the action enum in the tool schema." @@ -429,10 +438,11 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> f"{action} would go to the current target {mismatch!r}, not {requested_app.strip()!r} " "— input actions always hit the sticky target from the last capture/focus_app. " f"Call capture(app={requested_app.strip()!r}) or focus_app first, then retry.")}) - # delivery_mode / bring_to_front thread through every input action (background → foreground ladder). - res = spec.handler(backend, action, args, delivery_mode=args.get("delivery_mode"), - bring_to_front=bool(args.get("bring_to_front"))) - return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after"))) + # delivery_mode / bring_to_front thread through every input action (background → foreground ladder); + # read-only actions get the lease fence instead (delivery kwargs would leak into backend input calls). + res = spec.handler(backend, action, args, **(dict(delivery_mode=args.get("delivery_mode"), bring_to_front=bool(args.get("bring_to_front"))) + if spec.input else dict(fence=fence))) + return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after")), fence) # ── Response shaping ──────────────────────────────────────────────────────── def _classify_action_result(res: ActionResult) -> Dict[str, Any]: @@ -610,7 +620,8 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME "elements_file — read_file/search_files it, or pass app= to narrow scope)") return _text_capture_payload(v, "\n".join(lines), extra) -def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool) -> Any: +def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool, + fence: Callable[[], None] = lambda: None) -> Any: # No follow-up capture after a failed action: a normal-looking screenshot would suggest success. if not do_capture or not res.ok: return _text_response(res) @@ -623,6 +634,7 @@ def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_cap except Exception as e: logger.warning("follow-up capture failed: %s", e) return _text_response(res) + fence() resp, payload = _capture_response(cap), _action_payload(res) if isinstance(resp, dict) and resp.get("_multimodal"): # Keep the evidence/verdict contract visible alongside the image — it governs whether input may repeat. From fee40e76f8def89eec06c3feb2e0ed4227b0bdfb Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:07:24 -0700 Subject: [PATCH 16/55] fix(bot-desktop): lease reader fails closed on well-formed JSON of the wrong shape `[]`, `null` or `5` parsed fine and then raised AttributeError outside the except tuple; `{}` or an unknown holder read as "agent holds". Any of these is a torn or tampered file, and an untrusted lease must never let the agent act on a screen a human may be using. (cherry picked from commit dd77806ab619fcd55ce45698c5f3d739fdd63fa9) --- tests/tools/test_bot_desktop_lease.py | 7 +++++++ tools/bot_desktop/lease.py | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py index f1ada766cef1..50f948f46ad5 100644 --- a/tests/tools/test_bot_desktop_lease.py +++ b/tests/tools/test_bot_desktop_lease.py @@ -118,6 +118,13 @@ def test_unreadable_lease_file_fails_closed_and_takeover_keeps_the_agents_reason lease.release(profile_key=home) # a successful write repairs it assert lease.get(profile_key=home).holder == lease.AGENT + # Valid JSON of the wrong shape is just as untrustworthy as torn JSON: never read it as "agent holds". + for wrong_shape in ("[]", "null", "5", "{}", '{"holder": "root"}'): + path.write_text(wrong_shape, encoding="utf-8") + assert lease.get(profile_key=home).holder == lease.HUMAN, wrong_shape + lease.release(profile_key=home) + assert lease.get(profile_key=home).holder == lease.AGENT + lease.request_handoff("log in to the bank, 2FA on your phone", profile_key=home) held = lease.acquire("desk-1", profile_key=home) assert held.pending_handoff is None and held.reason == "log in to the bank, 2FA on your phone" diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py index f890677dc386..9aea7a63d89b 100644 --- a/tools/bot_desktop/lease.py +++ b/tools/bot_desktop/lease.py @@ -74,8 +74,13 @@ def _read(path: Path) -> Lease: return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file unreadable") try: data = json.loads(raw) + except ValueError: + data = None + if not isinstance(data, dict) or data.get("holder") not in (AGENT, HUMAN): + return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file corrupt") + try: return Lease(**{k: v for k, v in data.items() if k in Lease.__dataclass_fields__}) - except (ValueError, TypeError): + except TypeError: return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file corrupt") From e5800fd1457015c4bb8b75971d7ac118cc13bc7f Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:07:25 -0700 Subject: [PATCH 17/55] fix(display): display.thumbnail is suppressed while a human holds the lease MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Desktop polls thumbnails on a timer, so every connected client kept receiving frames of the screen a human had taken over — the same frames computer_use refuses to capture. Answer {data_url: null, suppressed: 'human_has_control'} without touching the framebuffer. (cherry picked from commit 9bb968026fe6563f793c4fb6aa67ce0577bc881e) --- tests/tui_gateway/test_display_methods.py | 29 +++++++++++++++++++++++ tui_gateway/methods_display.py | 6 ++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/tui_gateway/test_display_methods.py b/tests/tui_gateway/test_display_methods.py index d23cef507b1c..8a863821894e 100644 --- a/tests/tui_gateway/test_display_methods.py +++ b/tests/tui_gateway/test_display_methods.py @@ -31,3 +31,32 @@ def fake_install(*, ask_password, on_line, timeout_seconds=900.0): assert resp["result"]["started"], resp assert done.wait(5) assert seen["home"] == str(named) + + +@pytest.fixture +def _fresh_lease(): + from tools.bot_desktop import lease + lease._reset_for_tests() + yield lease + lease._reset_for_tests() + + +def _call(server, method, params): + return server.handle_request({"jsonrpc": "2.0", "id": 7, "method": method, "params": params}) + + +def test_thumbnail_is_suppressed_while_a_human_holds_the_lease(monkeypatch, _fresh_lease): + """The Desktop polls thumbnails on a timer; while a human drives the screen that grab would ship + whatever they are typing to every connected client, so it must not touch the framebuffer at all.""" + import tui_gateway.server as server + from tools.bot_desktop import thumbnail + + grabs = [] + monkeypatch.setattr(thumbnail, "thumbnail_data_url", lambda: grabs.append(1) or "data:image/jpeg;base64,SECRET") + _fresh_lease.acquire("viewer-1") + result = _call(server, "display.thumbnail", {})["result"] + assert result["data_url"] is None and result["suppressed"] == "human_has_control" + assert grabs == [], "the framebuffer was grabbed while a human held the lease" + _fresh_lease.release("viewer-1") + assert _call(server, "display.thumbnail", {})["result"]["data_url"].endswith("SECRET") + diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index 772668916974..aae515fe43f8 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -59,8 +59,12 @@ def _(rid, params: dict) -> dict: @method("display.thumbnail") @_profile_scoped def _(rid, params: dict) -> dict: - """One JPEG grab of the bot's screen (``data_url``: null while stopped). Read-only: no lease change.""" + """One JPEG grab of the bot's screen (``data_url``: null while stopped). Read-only: no lease change. + Suppressed while a human holds the lease — the frame may show what they are typing.""" try: + from tools.bot_desktop import lease as _bd_lease + if _bd_lease.human_holds(): + return _ok(rid, {"data_url": None, "suppressed": "human_has_control"}) from tools.bot_desktop.thumbnail import thumbnail_data_url return _ok(rid, {"data_url": thumbnail_data_url()}) except Exception as e: From 5dad306c8a744fedd96d760f144550a6e46d1507 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:07:25 -0700 Subject: [PATCH 18/55] fix(display): display.lease.release without a viewer_id no longer yanks another viewer's lease lease.release(None) skips the holder check, so a client that lost its viewer id (or any bare RPC) could take control away from whoever held it. Refuse with code viewer_mismatch unless params.force is set; display.stop and the CLI keep their unconditional release. (cherry picked from commit ae86da0a8d83ee60b538efb9f545d36213a3228c) --- tests/tui_gateway/test_display_methods.py | 13 +++++++++++++ tui_gateway/methods_display.py | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/tests/tui_gateway/test_display_methods.py b/tests/tui_gateway/test_display_methods.py index 8a863821894e..e6aa3287e86d 100644 --- a/tests/tui_gateway/test_display_methods.py +++ b/tests/tui_gateway/test_display_methods.py @@ -60,3 +60,16 @@ def test_thumbnail_is_suppressed_while_a_human_holds_the_lease(monkeypatch, _fre _fresh_lease.release("viewer-1") assert _call(server, "display.thumbnail", {})["result"]["data_url"].endswith("SECRET") + +def test_release_without_viewer_id_cannot_yank_another_viewers_lease(_fresh_lease): + """lease.release(None) skips the holder check, so a client that lost its viewer id (or a bare RPC) + must be refused unless it forces; a matching viewer id and force keep working.""" + import tui_gateway.server as server + + _fresh_lease.acquire("viewer-1") + refused = _call(server, "display.lease.release", {}) + assert refused["error"]["data"]["code"] == "viewer_mismatch" + assert _fresh_lease.get().holder == _fresh_lease.HUMAN + assert _call(server, "display.lease.release", {"viewer_id": "viewer-1"})["result"]["lease"]["holder"] == _fresh_lease.AGENT + _fresh_lease.acquire("viewer-2") + assert _call(server, "display.lease.release", {"force": True})["result"]["lease"]["holder"] == _fresh_lease.AGENT diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index aae515fe43f8..37d0248dd378 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -174,6 +174,11 @@ def _(rid, params: dict) -> dict: def _(rid, params: dict) -> dict: from tools.bot_desktop import lease as _bd_lease viewer_id = str(params.get("viewer_id") or "").strip() or None + # lease.release(None) skips the holder check; a client that lost its viewer id must not be able to + # yank control from whoever holds it unless it says so explicitly (force). + if viewer_id is None and not params.get("force") and _bd_lease.human_holds(): + return _err(rid, _DISPLAY_ERR, "viewer_id required to release another viewer's lease (or pass force: true)", + data={"code": "viewer_mismatch"}) lease = _bd_lease.release(viewer_id) return _ok(rid, {"lease": lease.as_dict()}) From 6d69254d8e231c19b06215d85f9d6f2a3b1a3c31 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:09:23 -0700 Subject: [PATCH 19/55] fix(computer-use): wait_for_human returns no_takeover when nobody answers the handoff wait_for_release polled until the full timeout (10 min by default) even when the holder never left AGENT, so an unseen request_handoff blocked the turn instead of letting the model chase the user in chat. After `grace` seconds (param, default 60, capped at the timeout) with the handoff still pending and no human holding, return {ok: false, code: no_takeover}. Once a human holds the screen the full timeout still applies to the hand-back. (cherry picked from commit 1402af036044a7191ec885fc8fa5c24206880909) --- tests/tools/test_computer_use_handoff_wait.py | 38 +++++++++++++++++++ tools/bot_desktop/lease.py | 12 +++++- tools/computer_use/handoff.py | 10 +++++ 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 tests/tools/test_computer_use_handoff_wait.py diff --git a/tests/tools/test_computer_use_handoff_wait.py b/tests/tools/test_computer_use_handoff_wait.py new file mode 100644 index 000000000000..29811d858054 --- /dev/null +++ b/tests/tools/test_computer_use_handoff_wait.py @@ -0,0 +1,38 @@ +"""wait_for_human answers an unanswered handoff early instead of blocking the whole timeout, and still +waits the full timeout once a human actually holds the screen.""" + +from __future__ import annotations + +import json +import threading +import time + +import pytest + +from tools.bot_desktop import lease +from tools.computer_use.handoff import handle_handoff + + +@pytest.fixture(autouse=True) +def _fresh_lease(): + lease._reset_for_tests() + yield + lease._reset_for_tests() + + +def test_wait_for_human_returns_no_takeover_when_nobody_answers_but_waits_out_a_real_takeover(): + handle_handoff("request_handoff", {"reason": "log in"}) + t0 = time.monotonic() + res = json.loads(handle_handoff("wait_for_human", {"seconds": 30, "grace": 0.2})) + assert res["code"] == "no_takeover" and res["state"]["pending_handoff"] == "log in" + assert time.monotonic() - t0 < 10, "an unanswered request must not run the full timeout" + + # A human takes over inside the grace window and hands back later: the wait outlives the grace. + def _take_then_release(): + time.sleep(0.1) + lease.acquire("viewer-1") + time.sleep(0.6) + lease.release("viewer-1") + threading.Thread(target=_take_then_release, daemon=True).start() + res = json.loads(handle_handoff("wait_for_human", {"seconds": 30, "grace": 0.3})) + assert res["ok"] and res["state"]["holder"] == lease.AGENT diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py index 9aea7a63d89b..33b12f200970 100644 --- a/tools/bot_desktop/lease.py +++ b/tools/bot_desktop/lease.py @@ -190,11 +190,19 @@ def wait_for_release(*, timeout: float, profile_key: Optional[str] = None) -> bo """Block until the agent holds the lease (and no handoff is pending) or ``timeout`` elapses. True when control is back with the agent. Polls the file so a release made by another process is seen; the local Condition just shortens the wait for same-process transitions.""" + return _wait_until(lambda lease: lease.holder == AGENT and lease.pending_handoff is None, timeout, profile_key) + + +def wait_for_takeover_or_release(*, timeout: float, profile_key: Optional[str] = None) -> bool: + """False when, after ``timeout``, the agent still holds with a handoff pending: nobody answered.""" + return _wait_until(lambda lease: lease.holder == HUMAN or lease.pending_handoff is None, timeout, profile_key) + + +def _wait_until(done: Callable[[Lease], bool], timeout: float, profile_key: Optional[str]) -> bool: path = _path(profile_key) deadline = time.monotonic() + timeout while True: - lease = _read(path) - if lease.holder == AGENT and lease.pending_handoff is None: + if done(_read(path)): return True remaining = deadline - time.monotonic() if remaining <= 0: diff --git a/tools/computer_use/handoff.py b/tools/computer_use/handoff.py index 11e30d788a7f..bdd38acbb1ad 100644 --- a/tools/computer_use/handoff.py +++ b/tools/computer_use/handoff.py @@ -18,6 +18,7 @@ HANDOFF_ACTIONS = frozenset({"request_handoff", "wait_for_human"}) _DEFAULT_WAIT_SECONDS = 600.0 _MAX_WAIT_SECONDS = 1800.0 +_DEFAULT_GRACE_SECONDS = 60.0 def handle_handoff(action: str, args: Dict[str, Any]) -> str: @@ -31,6 +32,15 @@ def handle_handoff(action: str, args: Dict[str, Any]) -> str: "computer_use action='wait_for_human' to block until they hand control back and " "re-capture before continuing — the screen state is whatever they left."}) timeout = min(_MAX_WAIT_SECONDS, max(1.0, float(args.get("seconds") or _DEFAULT_WAIT_SECONDS))) + grace = min(timeout, max(0.0, float(args.get("grace") or _DEFAULT_GRACE_SECONDS))) + # Nobody has taken over yet: give them `grace` to click Take over, then return so the model can chase + # the user in chat instead of blocking the whole timeout on a request nobody saw. Once a human holds + # the screen, wait the full timeout for the hand-back. + if not _lease.wait_for_takeover_or_release(timeout=grace): + state = _lease.get().as_dict() + return json.dumps({"ok": False, "action": action, "code": "no_takeover", "state": state, + "error": f"Nobody took over within {grace:.0f}s. Ask the user in chat to open Bots > Screen and " + "click Take over, then call wait_for_human again."}) released = _lease.wait_for_release(timeout=timeout) state = _lease.get().as_dict() if released: From 0dbc16852003a16e55278cd15f383f3cc8a55863 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:02:29 -0700 Subject: [PATCH 20/55] fix(bot-screen): a display ticket is not a gateway login on /api/ws /api/ws consumed any ticket and stamped its identity; display.observe mints provider "bot-desktop" tickets for viewers who may only be watching a screen, so one of those redeemed on /api/ws became a full authenticated session. Refuse bot-desktop tickets in _ws_auth_reason (reported as ticket_invalid, same as the display route refuses gateway tickets). (cherry picked from commit 753c3c35975fe1efab4a5bbc8bc6bbc9532b8521) --- hermes_cli/web_server_chat.py | 7 +++- .../hermes_cli/test_ws_auth_display_ticket.py | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/hermes_cli/test_ws_auth_display_ticket.py diff --git a/hermes_cli/web_server_chat.py b/hermes_cli/web_server_chat.py index 047bc867afb9..7196a3254f45 100644 --- a/hermes_cli/web_server_chat.py +++ b/hermes_cli/web_server_chat.py @@ -270,7 +270,12 @@ def _stamp_identity(info) -> None: return "no_credential", "none" try: - _stamp_identity(consume_ticket(ticket)) + info = consume_ticket(ticket) + if info.get("provider") == "bot-desktop": + # A display ticket admits one RFB bridge on /api/display/ws (a watch-only + # capability handed to a screen viewer); it must not double as a login here. + raise TicketInvalid("display ticket presented as a gateway login") + _stamp_identity(info) if protocol_ticket: # Select only the stable public protocol during accept. The # ticket-bearing protocol is a credential and must never be diff --git a/tests/hermes_cli/test_ws_auth_display_ticket.py b/tests/hermes_cli/test_ws_auth_display_ticket.py new file mode 100644 index 000000000000..626f2c106c38 --- /dev/null +++ b/tests/hermes_cli/test_ws_auth_display_ticket.py @@ -0,0 +1,37 @@ +"""A Bot Desktop display ticket admits one RFB bridge on ``/api/display/ws`` — it must never pass as a +full login on ``/api/ws`` (the reverse of ``test_display_ws_ticket``, which refuses gateway tickets there).""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from hermes_cli import web_server +import hermes_cli.web_server_chat as _web_server_chat +from hermes_cli.dashboard_auth.ws_tickets import _reset_for_tests, mint_ticket + + +@pytest.fixture +def gated_state(): + _reset_for_tests() + prev = getattr(web_server.app.state, "auth_required", None) + web_server.app.state.auth_required = True + yield + web_server.app.state.auth_required = prev + _reset_for_tests() + + +def _ws(ticket: str): + return SimpleNamespace( + query_params={"ticket": ticket}, headers={}, + client=SimpleNamespace(host="203.0.113.9"), url=SimpleNamespace(path="/api/ws")) + + +def test_display_ticket_is_refused_as_a_gateway_login(gated_state): + ticket = mint_ticket(user_id="display:v", provider="bot-desktop", + extra={"hermes_home": "/srv/hermes/bot-a", "viewer_id": "v"}) + ws = _ws(ticket) + reason, _credential = _web_server_chat._ws_auth_reason(ws) + assert reason == "ticket_invalid" + assert not hasattr(ws, "_hermes_auth_identity") From 99a16206a4e3491b9b1a51c4612a86f1c4e75636 Mon Sep 17 00:00:00 2001 From: Yags <166958865+whyyagswhy@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:39:01 -0400 Subject: [PATCH 21/55] fix(bot-screen): bound RFB clipboard buffering Reject oversized positive and extended clipboard lengths at the header. Process coalesced WebSocket data in bounded slices without rejecting valid multi-message frames. Includes fragmented-header and boundary regressions. Addresses the clipboard finding reported by carlotestor and corroborated by helix4u and other reviewers on #108914. (cherry picked from commit b1fbe363266e6d6fa3d73d2605fe1ca383607859) --- tests/tools/test_bot_desktop_rfb_limits.py | 40 ++++++++++++++++++++++ tools/bot_desktop/rfb_filter.py | 19 ++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/tools/test_bot_desktop_rfb_limits.py diff --git a/tests/tools/test_bot_desktop_rfb_limits.py b/tests/tools/test_bot_desktop_rfb_limits.py new file mode 100644 index 000000000000..9b5f55bfc2fa --- /dev/null +++ b/tests/tools/test_bot_desktop_rfb_limits.py @@ -0,0 +1,40 @@ +"""Untrusted RFB clipboard lengths are bounded without breaking stream framing.""" + +import pytest + +from tools.bot_desktop.rfb_filter import RfbClientFilter + +_HANDSHAKE = b"RFB 003.008\n\x01\x01" +_CLIPBOARD_LIMIT = 256 * 1024 + + +def clipboard_header(length): + return b"\x06\x00\x00\x00" + length.to_bytes(4, "big", signed=True) + + +@pytest.mark.parametrize("length", [_CLIPBOARD_LIMIT + 1, -_CLIPBOARD_LIMIT - 1, 2**31 - 1, -(2**31)]) +@pytest.mark.parametrize("holder", [False, True]) +def test_oversized_clipboard_is_rejected_at_header_without_waiting_for_payload(length, holder): + parser = RfbClientFilter(lambda: holder) + parser.feed(_HANDSHAKE) + header = clipboard_header(length) + for byte in header[:-1]: + assert parser.feed(bytes([byte])) == b"" + with pytest.raises(ValueError, match="clipboard"): + parser.feed(header[-1:]) + + +@pytest.mark.parametrize("extended", [False, True]) +@pytest.mark.parametrize("holder", [False, True]) +def test_bounded_clipboards_and_watch_requests_survive_fragmentation_and_large_coalesced_chunks(extended, holder): + parser = RfbClientFilter(lambda: holder) + assert parser.feed(_HANDSHAKE) == _HANDSHAKE + payload = b"x" * _CLIPBOARD_LIMIT + message = clipboard_header(-len(payload) if extended else len(payload)) + payload + refresh = b"\x03\x00" + b"\x00" * 8 + # A WebSocket frame can contain several valid messages, not just one. + stream = message + refresh + message + refresh + received = parser.feed(stream[:7]) + parser.feed(stream[7:]) + expected = (message + refresh) * 2 if holder else refresh * 2 + assert received == expected + assert parser.feed(refresh) == refresh diff --git a/tools/bot_desktop/rfb_filter.py b/tools/bot_desktop/rfb_filter.py index 804ad9d3fd42..4486df8a2bd1 100644 --- a/tools/bot_desktop/rfb_filter.py +++ b/tools/bot_desktop/rfb_filter.py @@ -30,6 +30,11 @@ _CLIENT_CUT_TEXT = 6 _FENCE = 248 +# Match TigerVNC's default MaxCutText. SetEncodings has a 16-bit count; every +# accepted message must fit in this buffer even when the WebSocket coalesces many. +_MAX_CUT_TEXT = 256 * 1024 +_MAX_BUFFER = max(8 + _MAX_CUT_TEXT, 4 + 4 * 0xFFFF) + class RfbClientFilter: """Feed client bytes with :meth:`feed`; get back the bytes allowed to reach Xvnc. @@ -44,6 +49,18 @@ def __init__(self, allow_input: Callable[[], bool]) -> None: self._handshake_left = 12 + 1 + 1 # version + security type + ClientInit(shared flag) def feed(self, chunk: bytes) -> bytes: + out = bytearray() + offset = 0 + while offset < len(chunk): + room = _MAX_BUFFER - len(self._buf) + if room <= 0: + raise ValueError("RFB client message exceeds buffer limit") + end = min(len(chunk), offset + room) + out += self._feed(chunk[offset:end]) + offset = end + return bytes(out) + + def _feed(self, chunk: bytes) -> bytes: self._buf += chunk out = bytearray() if self._handshake_left: @@ -84,6 +101,8 @@ def _message_length(self) -> int | None: return None n = int.from_bytes(self._buf[4:8], "big", signed=True) # Extended clipboard (RFB 3.8 + TigerVNC): negative length, |n| bytes follow. + if abs(n) > _MAX_CUT_TEXT: + raise ValueError("RFB clipboard exceeds 256 KiB limit") return 8 + abs(n) if t == _FENCE: if len(self._buf) < 9: From eb1799b71288b3387453a43246813c5ac76abd3f Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:03:16 -0700 Subject: [PATCH 22/55] refactor(bot-screen): keep only the RFB clipboard header cap The header check alone removes the unbounded buffering: every other client message type is fixed-size or bounded by a 16-bit count / one byte, so once ClientCutText is capped nothing can grow _buf past ~256 KiB + a partial frame. The feed()/_feed() slicing wrapper was a second layer over the same fact and its coalesced-frame test exercised behaviour the base parser already had. Rename the test file into the rfb_filter family. (cherry picked from commit b9bfa7de4c7e968347923d109c3ae68a66ceb37d) --- .../test_bot_desktop_rfb_filter_limits.py | 24 +++++++++++ tests/tools/test_bot_desktop_rfb_limits.py | 40 ------------------- tools/bot_desktop/rfb_filter.py | 19 ++------- 3 files changed, 27 insertions(+), 56 deletions(-) create mode 100644 tests/tools/test_bot_desktop_rfb_filter_limits.py delete mode 100644 tests/tools/test_bot_desktop_rfb_limits.py diff --git a/tests/tools/test_bot_desktop_rfb_filter_limits.py b/tests/tools/test_bot_desktop_rfb_filter_limits.py new file mode 100644 index 000000000000..018336d9f640 --- /dev/null +++ b/tests/tools/test_bot_desktop_rfb_filter_limits.py @@ -0,0 +1,24 @@ +"""A client-declared ClientCutText length is bounded at the header: the bridge must not buffer up to +2 GiB for a viewer that holds a ticket but no lease.""" + +import pytest + +from tools.bot_desktop.rfb_filter import _MAX_CUT_TEXT, RfbClientFilter + +_HANDSHAKE = b"RFB 003.008\n\x01\x01" + + +def clipboard_header(length): + return b"\x06\x00\x00\x00" + length.to_bytes(4, "big", signed=True) + + +@pytest.mark.parametrize("length", [_MAX_CUT_TEXT + 1, -_MAX_CUT_TEXT - 1, 2**31 - 1, -(2**31)]) +@pytest.mark.parametrize("holder", [False, True]) +def test_oversized_clipboard_is_rejected_at_header_without_waiting_for_payload(length, holder): + parser = RfbClientFilter(lambda: holder) + parser.feed(_HANDSHAKE) + header = clipboard_header(length) + for byte in header[:-1]: + assert parser.feed(bytes([byte])) == b"" + with pytest.raises(ValueError, match="clipboard"): + parser.feed(header[-1:]) diff --git a/tests/tools/test_bot_desktop_rfb_limits.py b/tests/tools/test_bot_desktop_rfb_limits.py deleted file mode 100644 index 9b5f55bfc2fa..000000000000 --- a/tests/tools/test_bot_desktop_rfb_limits.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Untrusted RFB clipboard lengths are bounded without breaking stream framing.""" - -import pytest - -from tools.bot_desktop.rfb_filter import RfbClientFilter - -_HANDSHAKE = b"RFB 003.008\n\x01\x01" -_CLIPBOARD_LIMIT = 256 * 1024 - - -def clipboard_header(length): - return b"\x06\x00\x00\x00" + length.to_bytes(4, "big", signed=True) - - -@pytest.mark.parametrize("length", [_CLIPBOARD_LIMIT + 1, -_CLIPBOARD_LIMIT - 1, 2**31 - 1, -(2**31)]) -@pytest.mark.parametrize("holder", [False, True]) -def test_oversized_clipboard_is_rejected_at_header_without_waiting_for_payload(length, holder): - parser = RfbClientFilter(lambda: holder) - parser.feed(_HANDSHAKE) - header = clipboard_header(length) - for byte in header[:-1]: - assert parser.feed(bytes([byte])) == b"" - with pytest.raises(ValueError, match="clipboard"): - parser.feed(header[-1:]) - - -@pytest.mark.parametrize("extended", [False, True]) -@pytest.mark.parametrize("holder", [False, True]) -def test_bounded_clipboards_and_watch_requests_survive_fragmentation_and_large_coalesced_chunks(extended, holder): - parser = RfbClientFilter(lambda: holder) - assert parser.feed(_HANDSHAKE) == _HANDSHAKE - payload = b"x" * _CLIPBOARD_LIMIT - message = clipboard_header(-len(payload) if extended else len(payload)) + payload - refresh = b"\x03\x00" + b"\x00" * 8 - # A WebSocket frame can contain several valid messages, not just one. - stream = message + refresh + message + refresh - received = parser.feed(stream[:7]) + parser.feed(stream[7:]) - expected = (message + refresh) * 2 if holder else refresh * 2 - assert received == expected - assert parser.feed(refresh) == refresh diff --git a/tools/bot_desktop/rfb_filter.py b/tools/bot_desktop/rfb_filter.py index 4486df8a2bd1..b4d4a63e2c42 100644 --- a/tools/bot_desktop/rfb_filter.py +++ b/tools/bot_desktop/rfb_filter.py @@ -30,10 +30,9 @@ _CLIENT_CUT_TEXT = 6 _FENCE = 248 -# Match TigerVNC's default MaxCutText. SetEncodings has a 16-bit count; every -# accepted message must fit in this buffer even when the WebSocket coalesces many. +# TigerVNC's default MaxCutText. The length is client-declared (int32); without a cap a watcher +# with a ticket but no lease could make the bridge buffer ~2 GiB waiting for a payload. _MAX_CUT_TEXT = 256 * 1024 -_MAX_BUFFER = max(8 + _MAX_CUT_TEXT, 4 + 4 * 0xFFFF) class RfbClientFilter: @@ -49,18 +48,6 @@ def __init__(self, allow_input: Callable[[], bool]) -> None: self._handshake_left = 12 + 1 + 1 # version + security type + ClientInit(shared flag) def feed(self, chunk: bytes) -> bytes: - out = bytearray() - offset = 0 - while offset < len(chunk): - room = _MAX_BUFFER - len(self._buf) - if room <= 0: - raise ValueError("RFB client message exceeds buffer limit") - end = min(len(chunk), offset + room) - out += self._feed(chunk[offset:end]) - offset = end - return bytes(out) - - def _feed(self, chunk: bytes) -> bytes: self._buf += chunk out = bytearray() if self._handshake_left: @@ -102,7 +89,7 @@ def _message_length(self) -> int | None: n = int.from_bytes(self._buf[4:8], "big", signed=True) # Extended clipboard (RFB 3.8 + TigerVNC): negative length, |n| bytes follow. if abs(n) > _MAX_CUT_TEXT: - raise ValueError("RFB clipboard exceeds 256 KiB limit") + raise ValueError("clipboard message too large") return 8 + abs(n) if t == _FENCE: if len(self._buf) < 9: From 329a16062c73a27e195add73af6cf536ca231592 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:04:50 -0700 Subject: [PATCH 23/55] fix(bot-screen): a viewer who handed back is not evicted by a later takeover _bridge remembered "this viewer held control at some point" and never forgot it, so after a hand-back to the agent a takeover by another human closed the ex-holder's socket with 4000 control-taken although they were a plain watcher by then. The eviction rule is now _should_evict(held, lease, viewer_id): a lease held by the agent clears the memory; only a takeover while this viewer still held evicts. (cherry picked from commit 178abd0f87f980e603717b1eca39e4344816f7aa) --- hermes_cli/web_routers/display.py | 22 ++++++++++++++----- .../test_display_ws_drop_keeps_lease.py | 13 +++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py index 541be828eb7c..577036294ce4 100644 --- a/hermes_cli/web_routers/display.py +++ b/hermes_cli/web_routers/display.py @@ -33,6 +33,21 @@ _CLOSE_PROTOCOL = 1003 +def _should_evict(held: dict, lease, viewer_id: str) -> bool: + """A viewer that held control during this connection and lost it to ANOTHER human is kicked so its + UI repaints; a plain hand-back to the agent, pure watchers and the new holder stay connected. The + hand-back also forgets that this viewer ever held: after it, they are a plain watcher again and a + later takeover by someone else must not evict them. ``held`` is the per-connection memory.""" + from tools.bot_desktop import lease as _lease + if lease.holder != _lease.HUMAN: + held["ever"] = False + return False + if lease.viewer_id == viewer_id: + held["ever"] = True + return False + return bool(held["ever"]) + + def _consume_display_ticket(ws: WebSocket) -> Optional[dict]: from hermes_cli.dashboard_auth.ws_tickets import TicketInvalid, consume_ticket ticket = ws.query_params.get("display_ticket", "") @@ -86,14 +101,9 @@ async def _bridge(ws: WebSocket, info: dict) -> None: held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)} def _on_lease(key: str, lease) -> None: - # A viewer that held control during this connection and lost it to ANOTHER human is kicked so - # its UI repaints; a plain hand-back to the agent, pure watchers and the new holder stay connected. if key != profile_key: return - mine = lease.holder == _lease.HUMAN and lease.viewer_id == viewer_id - if mine: - held["ever"] = True - elif held["ever"] and lease.holder == _lease.HUMAN: + if _should_evict(held, lease, viewer_id): loop.call_soon_threadsafe(evicted.set) unsubscribe = _lease.on_change(_on_lease) diff --git a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py index e1ea23d7329c..00be45e0c181 100644 --- a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py +++ b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py @@ -60,3 +60,16 @@ def test_only_a_clean_viewer_close_hands_the_screen_back(monkeypatch, close_code after = asyncio.run(_bridge_once(close_code, home)) lease._reset_for_tests() assert (after.holder == lease.HUMAN) is human_keeps_control, after + + +def test_ex_holder_who_handed_back_is_not_evicted_by_a_later_takeover(): + """desk-1 holds, hands back to the agent, then desk-2 takes over: desk-1 is a plain watcher again + and must stay connected; only a takeover WHILE desk-1 held (or believed it held) kicks it.""" + held = {"ever": False} + assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-1"), "desk-1") is False + assert display._should_evict(held, lease.Lease(holder=lease.AGENT), "desk-1") is False + assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-2"), "desk-1") is False + + held = {"ever": False} + display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-1"), "desk-1") + assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-2"), "desk-1") is True From 46b5c7746a6168162d0ff185ea1b301dd35150ad Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:04:59 -0700 Subject: [PATCH 24/55] test(bot-screen): a 1005 close keeps the human's lease noVNC's code-less socket.close() and some proxies both surface as 1005 on the server, so the bridge cannot tell a deliberate pane close from a drop and must keep the lease. The Desktop closes with an explicit 1000 on unmount; this row pins the server side of that contract. (cherry picked from commit a62a7429f2b925d93d2122b7fa7f235144b8de44) --- tests/hermes_cli/test_display_ws_drop_keeps_lease.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py index 00be45e0c181..b530648164d3 100644 --- a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py +++ b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py @@ -52,7 +52,9 @@ async def _xvnc(reader, writer): # a silent framebuffer return lease.get(profile_key=home) -@pytest.mark.parametrize(("close_code", "human_keeps_control"), [(1006, True), (1000, False)]) +# 1005 (no status code) is what noVNC's code-less socket.close() AND some proxies produce on a drop, so +# the server keeps the lease; the Desktop sends an explicit 1000 when the pane is closed on purpose. +@pytest.mark.parametrize(("close_code", "human_keeps_control"), [(1006, True), (1005, True), (1000, False)]) def test_only_a_clean_viewer_close_hands_the_screen_back(monkeypatch, close_code, human_keeps_control): lease._reset_for_tests() with tempfile.TemporaryDirectory() as home: From 135c28f802d6b3d863bcc64bc680d55635f72b17 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:05:46 -0700 Subject: [PATCH 25/55] perf(bot-screen): bridge caches the lease input decision RfbClientFilter consulted lease.viewer_may_send_input per client message, which stats and reads lease.json on the event loop for every pointer move. The bridge now keeps one boolean, refreshed from the in-process on_change listener (same-process takeovers apply to the very next message) and by a file re-read at most every 250 ms (a takeover written by another process is seen within one interval). Test: a foreign takeover stops input in < 0.5 s (fails with the interval set to 5 s). (cherry picked from commit dcafcb2448c9668c80f2c449a4cda5ad95f82bab) --- hermes_cli/web_routers/display.py | 19 ++++++- .../test_display_ws_drop_keeps_lease.py | 53 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py index 577036294ce4..4ccb2721893f 100644 --- a/hermes_cli/web_routers/display.py +++ b/hermes_cli/web_routers/display.py @@ -31,6 +31,7 @@ _CLOSE_BAD_TICKET = 4401 _CLOSE_NOT_ALLOWED = 4403 _CLOSE_PROTOCOL = 1003 +_LEASE_REFRESH_S = 0.25 def _should_evict(held: dict, lease, viewer_id: str) -> bool: @@ -99,15 +100,31 @@ async def _bridge(ws: WebSocket, info: dict) -> None: loop = asyncio.get_running_loop() evicted = asyncio.Event() held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)} + # Input gate cache: reading lease.json per client message (a stat + read on the event loop for + # every pointer move) is replaced by a decision refreshed on this process's on_change callback + # and by a file re-read at most every _LEASE_REFRESH_S, so another process's takeover still lands. + allowed = {"input": held["ever"], "at": loop.time()} + + def _refresh_allowed(lease=None) -> None: + if lease is None: + lease = _lease.get(profile_key=profile_home) + allowed["input"] = lease.holder == _lease.HUMAN and lease.viewer_id == viewer_id + allowed["at"] = loop.time() + + def _may_send_input() -> bool: + if loop.time() - allowed["at"] > _LEASE_REFRESH_S: + _refresh_allowed() + return allowed["input"] def _on_lease(key: str, lease) -> None: if key != profile_key: return + loop.call_soon_threadsafe(_refresh_allowed, lease) if _should_evict(held, lease, viewer_id): loop.call_soon_threadsafe(evicted.set) unsubscribe = _lease.on_change(_on_lease) - rfb_filter = RfbClientFilter(lambda: _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)) + rfb_filter = RfbClientFilter(_may_send_input) viewer_closed = asyncio.Event() diff --git a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py index b530648164d3..2bc98c15dba0 100644 --- a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py +++ b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py @@ -75,3 +75,56 @@ def test_ex_holder_who_handed_back_is_not_evicted_by_a_later_takeover(): held = {"ever": False} display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-1"), "desk-1") assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-2"), "desk-1") is True + + +class _OpenWs(_Ws): + """Stays open until ``finish`` is set, then reports a clean close.""" + + def __init__(self): + super().__init__(1000) + self.finish = asyncio.Event() + + async def receive(self): + await self.finish.wait() + return {"type": "websocket.disconnect", "code": self._code} + + +def test_a_takeover_made_by_another_process_stops_input_within_the_refresh_interval(monkeypatch): + """The bridge caches the input decision instead of reading lease.json per message; a takeover + written by ANOTHER process (no in-process listener fires) must still be seen quickly.""" + from tools.bot_desktop import rfb_filter + captured = {} + + class _Filter(rfb_filter.RfbClientFilter): + def __init__(self, allow_input): + super().__init__(allow_input) + captured["allow"] = allow_input + monkeypatch.setattr(rfb_filter, "RfbClientFilter", _Filter) + + async def _run(home: str) -> float: + sock_dir = os.path.join(home, "bot-desktop") + os.makedirs(sock_dir, exist_ok=True) + server = await asyncio.start_unix_server(lambda r, w: None, path=os.path.join(sock_dir, "rfb.sock")) + ws = _OpenWs() + task = asyncio.create_task(display._bridge(ws, {"hermes_home": home, "viewer_id": "desk-1"})) + try: + while "allow" not in captured: + await asyncio.sleep(0.01) + assert captured["allow"]() is True + # Another process takes over: the file changes, no listener in this process is told. + lease._write(lease._path(home), lease.Lease(holder=lease.HUMAN, viewer_id="desk-2", epoch=2)) + t0 = asyncio.get_running_loop().time() + while captured["allow"]() and asyncio.get_running_loop().time() - t0 < 2.0: + await asyncio.sleep(0.02) + return asyncio.get_running_loop().time() - t0 + finally: + ws.finish.set() + await task + server.close() + + lease._reset_for_tests() + with tempfile.TemporaryDirectory() as home: + lease.acquire("desk-1", profile_key=home) + elapsed = asyncio.run(_run(home)) + lease._reset_for_tests() + assert elapsed < 0.5, elapsed From bd9b4d5e99c861dd96e7c1f1d9dfafadab63b3e9 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:06:29 -0700 Subject: [PATCH 26/55] fix(bot-screen): serialise the XAUTHORITY swap around thumbnail grabs thumbnail_data_url swaps the process-wide os.environ["XAUTHORITY"] around ImageGrab.grab; a multiplexed gateway thumbnails several profiles from worker threads at once, so one grab could run with another profile's cookie (Xlib auth failure or the wrong screen) and the restore left the wrong value behind. A module lock serialises swap+grab+restore. (cherry picked from commit be789c85fdbae869a31d63f30a3434f03d350a50) --- tests/tools/test_bot_desktop_thumbnail.py | 41 +++++++++++++++++++++++ tools/bot_desktop/thumbnail.py | 25 ++++++++------ 2 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 tests/tools/test_bot_desktop_thumbnail.py diff --git a/tests/tools/test_bot_desktop_thumbnail.py b/tests/tools/test_bot_desktop_thumbnail.py new file mode 100644 index 000000000000..73f742c69afa --- /dev/null +++ b/tests/tools/test_bot_desktop_thumbnail.py @@ -0,0 +1,41 @@ +"""``thumbnail_data_url`` swaps the process-wide XAUTHORITY around the grab; two profiles grabbed on +worker threads at once must each see their own cookie file, not the other's.""" + +from __future__ import annotations + +import os +import threading + +from PIL import Image, ImageGrab + +from tools.bot_desktop import runtime, thumbnail + + +def test_concurrent_grabs_each_see_their_own_xauthority(monkeypatch): + envs = {"a": {"DISPLAY": ":91", "XAUTHORITY": "/tmp/xauth-a"}, + "b": {"DISPLAY": ":92", "XAUTHORITY": "/tmp/xauth-b"}} + local = threading.local() + monkeypatch.setattr(runtime, "published_env", lambda: envs[local.profile]) + monkeypatch.setattr(runtime, "_launcher_pid", lambda: 4242) + monkeypatch.delenv("XAUTHORITY", raising=False) + seen = {} + start = threading.Barrier(2) + + def fake_grab(xdisplay=None): + seen[xdisplay] = os.environ.get("XAUTHORITY") + threading.Event().wait(0.05) # hold the env long enough for the other thread to collide + return Image.new("RGB", (8, 8)) + monkeypatch.setattr(ImageGrab, "grab", fake_grab) + + def worker(profile): + local.profile = profile + start.wait() + thumbnail.thumbnail_data_url() + + threads = [threading.Thread(target=worker, args=(p,)) for p in envs] + for t in threads: + t.start() + for t in threads: + t.join(5) + assert seen == {":91": "/tmp/xauth-a", ":92": "/tmp/xauth-b"} + assert "XAUTHORITY" not in os.environ diff --git a/tools/bot_desktop/thumbnail.py b/tools/bot_desktop/thumbnail.py index ee41e6ffbc9b..eeb61b57892b 100644 --- a/tools/bot_desktop/thumbnail.py +++ b/tools/bot_desktop/thumbnail.py @@ -9,11 +9,13 @@ import base64 import io import os +import threading from typing import Optional from tools.bot_desktop import runtime THUMB_MAX = (960, 600) +_grab_lock = threading.Lock() def thumbnail_data_url(max_size: tuple[int, int] = THUMB_MAX, quality: int = 72) -> Optional[str]: @@ -25,16 +27,19 @@ def thumbnail_data_url(max_size: tuple[int, int] = THUMB_MAX, quality: int = 72) from PIL import ImageGrab # Pillow is a hard dependency; import lazily to keep status calls cheap # Xlib reads XAUTHORITY from the process env; the launcher publishes a per-profile cookie file. - previous = os.environ.get("XAUTHORITY") - if env.get("XAUTHORITY"): - os.environ["XAUTHORITY"] = env["XAUTHORITY"] - try: - image = ImageGrab.grab(xdisplay=display) - finally: - if previous is None: - os.environ.pop("XAUTHORITY", None) - else: - os.environ["XAUTHORITY"] = previous + # The swap is process-wide, so two profiles grabbed on worker threads at once serialise here or + # one would grab with the other's cookie and restore the wrong value. + with _grab_lock: + previous = os.environ.get("XAUTHORITY") + if env.get("XAUTHORITY"): + os.environ["XAUTHORITY"] = env["XAUTHORITY"] + try: + image = ImageGrab.grab(xdisplay=display) + finally: + if previous is None: + os.environ.pop("XAUTHORITY", None) + else: + os.environ["XAUTHORITY"] = previous image.thumbnail(max_size) buf = io.BytesIO() image.convert("RGB").save(buf, "JPEG", quality=quality, optimize=True) From 732723cf493a64ce935f79f4b29f8e88057f116f Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:11:25 -0700 Subject: [PATCH 27/55] fix(bot-screen): server-minted viewer ids; lease snapshots carry a hash, not the id display.observe took viewer_id from the client and display.status handed every client the holder's viewer_id, while the lease authorised input and release on string equality: any authenticated client could read the holder's id, mint an observe ticket with it and co-drive or release their lease. observe now mints viewer_id = secrets.token_urlsafe(16) and returns it; a requested id is honoured only when this same connection minted it earlier (a reconnecting pane keeps its lease), tracked per transport in a weak dict. Every lease payload leaving the gateway (display.status/start/stop/observe snapshots and the display.lease broadcast) goes through _lease_view, which replaces viewer_id with null plus viewer_hash = sha256(id)[:12] so the Desktop can still tell whether it is the holder. (cherry picked from commit 52ff67ab409ad547c05e8a4244eb6c24513adf65) --- tests/tui_gateway/test_display_methods.py | 52 ++++++++++++++++++++++- tui_gateway/methods_display.py | 42 ++++++++++++++++-- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/tests/tui_gateway/test_display_methods.py b/tests/tui_gateway/test_display_methods.py index e6aa3287e86d..67a6f207f081 100644 --- a/tests/tui_gateway/test_display_methods.py +++ b/tests/tui_gateway/test_display_methods.py @@ -1,11 +1,15 @@ -"""display.install runs its worker inside the caller's profile scope.""" +"""display.install runs its worker inside the caller's profile scope; display.observe mints the viewer identity.""" from __future__ import annotations +import hashlib +import json import threading import pytest +from hermes_cli.dashboard_auth import ws_tickets + def test_install_worker_keeps_the_requested_profile_scope(tmp_path, monkeypatch): from hermes_constants import get_hermes_home @@ -73,3 +77,49 @@ def test_release_without_viewer_id_cannot_yank_another_viewers_lease(_fresh_leas assert _call(server, "display.lease.release", {"viewer_id": "viewer-1"})["result"]["lease"]["holder"] == _fresh_lease.AGENT _fresh_lease.acquire("viewer-2") assert _call(server, "display.lease.release", {"force": True})["result"]["lease"]["holder"] == _fresh_lease.AGENT + +def _rpc(server, method, params): + return server.handle_request({"jsonrpc": "2.0", "id": 7, "method": method, "params": params}) + + +def test_observe_mints_the_viewer_id_and_status_never_discloses_the_holder(monkeypatch, tmp_path): + """A client cannot choose its viewer id (it would impersonate the holder and co-drive or release + their lease), and no snapshot or broadcast carries the raw holder id — only a hash the holder + itself can match.""" + from tools.bot_desktop import lease, runtime + import tui_gateway.server as server + + monkeypatch.setattr(runtime, "rfb_socket_path", lambda: tmp_path / "rfb.sock") + lease._reset_for_tests() + broadcasts = [] + monkeypatch.setattr(server, "_broadcast_global_event", lambda ev, payload=None: broadcasts.append((ev, payload))) + try: + observed = _rpc(server, "display.observe", {"viewer_id": "victim"})["result"] + assert observed["viewer_id"] != "victim" + assert observed["viewer_id"] and len(observed["viewer_id"]) >= 16 + assert ws_tickets.consume_ticket(observed["ticket"])["viewer_id"] == observed["viewer_id"] + holder = observed["viewer_id"] + + # Only the connection that minted an id may reuse it (a reconnecting pane keeps its lease). + class _Peer: + def write(self, obj): + return True + mine, other = _Peer(), _Peer() + with_mine = server.dispatch({"jsonrpc": "2.0", "id": 8, "method": "display.observe", "params": {}}, mine)["result"] + again = server.dispatch({"jsonrpc": "2.0", "id": 9, "method": "display.observe", + "params": {"viewer_id": with_mine["viewer_id"]}}, mine)["result"] + assert again["viewer_id"] == with_mine["viewer_id"] + stolen = server.dispatch({"jsonrpc": "2.0", "id": 10, "method": "display.observe", + "params": {"viewer_id": with_mine["viewer_id"]}}, other)["result"] + assert stolen["viewer_id"] != with_mine["viewer_id"] + + _rpc(server, "display.status", {}) # installs the broadcast listener + lease.acquire(holder) + status = _rpc(server, "display.status", {})["result"] + assert status["lease"]["holder"] == lease.HUMAN + assert holder not in json.dumps(status) + assert status["lease"]["viewer_hash"] == hashlib.sha256(holder.encode()).hexdigest()[:12] + lease_events = [p for ev, p in broadcasts if ev == "display.lease"] + assert lease_events and all(holder not in json.dumps(p) for p in lease_events) + finally: + lease._reset_for_tests() diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index 37d0248dd378..38e4fba69762 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -15,6 +15,7 @@ import logging import threading +import weakref from .method_ctx import HandlerRegistry, bind_module @@ -27,11 +28,23 @@ _lease_listener_installed = threading.Event() +def _lease_view(lease) -> dict: + """The lease as clients may see it: the holder's viewer id is a capability (whoever presents it + co-drives or releases the lease), so it is replaced by a short hash the holder can match against + its own id to know it is the one in control.""" + import hashlib + d = lease.as_dict() + vid = d.pop("viewer_id") + d["viewer_id"] = None + d["viewer_hash"] = hashlib.sha256(vid.encode()).hexdigest()[:12] if vid else None + return d + + def _display_snapshot() -> dict: from hermes_constants import hermes_home_key from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime st = _bd_runtime.status() - return {**st.as_dict(), "lease": _bd_lease.get().as_dict(), "profile_key": hermes_home_key()} + return {**st.as_dict(), "lease": _lease_view(_bd_lease.get()), "profile_key": hermes_home_key()} def _install_lease_listener() -> None: @@ -41,7 +54,7 @@ def _install_lease_listener() -> None: from tools.bot_desktop import lease as _bd_lease def _on_change(profile_key: str, lease) -> None: - _broadcast_global_event("display.lease", {"profile_key": profile_key, "lease": lease.as_dict()}) + _broadcast_global_event("display.lease", {"profile_key": profile_key, "lease": _lease_view(lease)}) _bd_lease.on_change(_on_change) _lease_listener_installed.set() # only once the subscription exists, or a failed import would silence every client @@ -95,18 +108,39 @@ def _(rid, params: dict) -> dict: return _err(rid, _DISPLAY_ERR, str(e)) +# viewer ids minted per connection (keyed by the transport that asked), so a reconnecting pane can +# keep its identity — and its lease — while nobody can claim an id minted for another connection. +_minted_viewer_ids: "weakref.WeakKeyDictionary[object, set[str]]" = weakref.WeakKeyDictionary() + + +def _mint_viewer_id(requested: str) -> str: + """Server-minted viewer identity. ``requested`` is honoured only when THIS connection minted it + earlier; anything else (including a holder id read off display.status) gets a fresh id.""" + import secrets + try: + mine = _minted_viewer_ids.setdefault(current_transport(), set()) + except TypeError: # stdio / slotted transports cannot be weakly referenced: always mint + mine = set() + if requested in mine: + return requested + viewer_id = secrets.token_urlsafe(16) + mine.add(viewer_id) + return viewer_id + + @method("display.observe") @_profile_scoped def _(rid, params: dict) -> dict: """Mint a single-use, 30 s ticket for ``/api/display/ws``. The ticket carries the profile home so - the bridge dials THIS profile's RFB socket, and the viewer id so the lease can name the holder.""" + the bridge dials THIS profile's RFB socket, and a server-minted viewer id (returned to the caller, + who passes it to ``display.lease.acquire`` / ``release``) so the lease can name the holder.""" from hermes_constants import get_hermes_home from hermes_cli.dashboard_auth.ws_tickets import mint_ticket from tools.bot_desktop import runtime as _bd_runtime try: if _bd_runtime.rfb_socket_path() is None: return _err(rid, _DISPLAY_ERR, "this profile's Bot Desktop is not running; call display.start first") - viewer_id = str(params.get("viewer_id") or "").strip() or f"viewer-{rid}" + viewer_id = _mint_viewer_id(str(params.get("viewer_id") or "").strip()) ticket = mint_ticket(user_id=f"display:{viewer_id}", provider="bot-desktop", extra={"hermes_home": str(get_hermes_home()), "viewer_id": viewer_id}) return _ok(rid, {"ticket": ticket, "path": "/api/display/ws", "viewer_id": viewer_id, From 9a6126ceefdf21b5327a5c2f9f07ffc53136b6af Mon Sep 17 00:00:00 2001 From: Yags <166958865+whyyagswhy@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:39:24 -0400 Subject: [PATCH 28/55] fix(bot-screen): release stranded control through CLI stop Match the existing gateway stop contract on supported hosts, including when the desktop has already exited. Keep the Linux lease import off unsupported hosts. Native Linux test exercises the CLI parser and real persisted lease. Addresses the CLI recovery finding from MrD1az and subsequent reviewers on #108914. (cherry picked from commit 0b361bf11b45ea9667369214b93919fad4c1b0f5) --- hermes_cli/subcommands/computer_use_screen.py | 3 +++ .../test_bot_desktop_screen_stop.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/hermes_cli/test_bot_desktop_screen_stop.py diff --git a/hermes_cli/subcommands/computer_use_screen.py b/hermes_cli/subcommands/computer_use_screen.py index 5c2dc5260638..236c6c3f5a08 100644 --- a/hermes_cli/subcommands/computer_use_screen.py +++ b/hermes_cli/subcommands/computer_use_screen.py @@ -46,6 +46,9 @@ def _screen_start(args) -> int: def _screen_stop(args) -> int: from tools.bot_desktop import runtime + if runtime.is_supported_host(): + from tools.bot_desktop import lease + lease.release() print("Bot Desktop: stopped" if runtime.stop() else "Bot Desktop: was not running") return 0 diff --git a/tests/hermes_cli/test_bot_desktop_screen_stop.py b/tests/hermes_cli/test_bot_desktop_screen_stop.py new file mode 100644 index 000000000000..33f661aed507 --- /dev/null +++ b/tests/hermes_cli/test_bot_desktop_screen_stop.py @@ -0,0 +1,19 @@ +"""The documented CLI stop is also recovery for a disconnected viewer's lease.""" + +import argparse + +from hermes_cli.subcommands.computer_use_screen import build_screen_parser +import pytest + + +@pytest.mark.linux_only +def test_screen_stop_hands_back_even_when_the_desktop_has_already_exited(): + from tools.bot_desktop import lease + + parser = argparse.ArgumentParser() + build_screen_parser(parser.add_subparsers(), lambda sub, help_text: sub.add_argument('--json', action='store_true')) + lease.acquire('disconnected-viewer') + args = parser.parse_args(['screen', 'stop']) + assert args.screen_func(args) == 0 + assert lease.get().holder == lease.AGENT + assert lease.wait_for_release(timeout=0) From bd3e726e02066618d7f343d0a14d84a112421edb Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:08:58 -0700 Subject: [PATCH 29/55] fix(bot-screen): Fedora per-binary package names, xprop required, binary->package map Fedora split xorg-x11-server-utils and xorg-x11-utils into per-binary packages (F35) and retired the umbrellas; dnf5 refuses the whole transaction on one unknown name, so `hermes computer-use screen install` was a no-op on Fedora. The dnf list now names xsetroot/xset/xdpyinfo/ xprop/setxkbmap directly. xprop (the launcher's WM-ready wait) joins REQUIRED_BINARIES and every distro list; apt gains x11-xkb-utils (setxkbmap) and pacman gains dbus (dbus-run-session), both previously reached only via transitive deps. BINARY_PACKAGES documents which package ships each binary per manager so a test can hold the lists to it. (cherry picked from commit dc5c4d70f7a0fd355fe21c96071513d8b5ae3744) --- tests/tools/test_bot_desktop_runtime.py | 12 +++++++++++ tools/bot_desktop/runtime.py | 28 ++++++++++++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/tools/test_bot_desktop_runtime.py b/tests/tools/test_bot_desktop_runtime.py index ab2080900459..31c28cba9534 100644 --- a/tests/tools/test_bot_desktop_runtime.py +++ b/tests/tools/test_bot_desktop_runtime.py @@ -4,9 +4,21 @@ import sys +import pytest + from tools.bot_desktop import runtime, thumbnail +@pytest.mark.parametrize("pm", sorted(runtime.PACKAGES)) +def test_every_required_binary_maps_to_an_installed_package(pm): + """Each binary the launcher execs must come from a package the distro list actually installs; dnf5 + refuses the whole transaction on one retired name, so the map is the contract, not the list.""" + mapping = runtime.BINARY_PACKAGES[pm] + assert set(mapping) == set(runtime.REQUIRED_BINARIES) + assert set(mapping.values()) <= set(runtime.PACKAGES[pm]) + assert not {"xorg-x11-server-utils", "xorg-x11-utils"} & set(runtime.PACKAGES["dnf"]), "retired on Fedora" + + def test_no_running_screen_returns_none_without_grabbing(monkeypatch): monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":99"}) monkeypatch.setattr(runtime, "_launcher_pid", lambda: None) diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py index 701349ca70d6..e6ecb8e6a0aa 100644 --- a/tools/bot_desktop/runtime.py +++ b/tools/bot_desktop/runtime.py @@ -37,16 +37,34 @@ # Binaries the launcher execs; the package hint is per distro family. REQUIRED_BINARIES = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session", - "xauth", "xdpyinfo", "setxkbmap") + "xauth", "xdpyinfo", "setxkbmap", "xprop") + +# Which package in each distro list ships each required binary. Fedora retired the xorg-x11-utils / +# xorg-x11-server-utils umbrellas (per-binary packages since F35) and dnf5 refuses the whole transaction on +# one unknown name, so every binary must map to a package that still resolves; the test suite checks that +# each mapped package is in PACKAGES for its manager. +BINARY_PACKAGES = { + "apt": {"Xvnc": "tigervnc-standalone-server", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", + "xfdesktop": "xfdesktop4", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11", + "xauth": "xauth", "xdpyinfo": "x11-utils", "setxkbmap": "x11-xkb-utils", "xprop": "x11-utils"}, + "dnf": {"Xvnc": "tigervnc-server-minimal", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", + "xfdesktop": "xfdesktop", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11", + "xauth": "xorg-x11-xauth", "xdpyinfo": "xdpyinfo", "setxkbmap": "setxkbmap", "xprop": "xprop"}, + "pacman": {"Xvnc": "tigervnc", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", "xfdesktop": "xfdesktop", + "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus", + "xauth": "xorg-xauth", "xdpyinfo": "xorg-xdpyinfo", "setxkbmap": "xorg-setxkbmap", "xprop": "xorg-xprop"}, +} PACKAGES = { "apt": ["tigervnc-standalone-server", "xfce4-panel", "xfwm4", "xfdesktop4", "xfce4-settings", - "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "xauth", "fonts-dejavu-core"], + "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "x11-xkb-utils", "xauth", + "fonts-dejavu-core"], "dnf": ["tigervnc-server-minimal", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", - "xfce4-terminal", "dbus-x11", "xorg-x11-server-utils", "xorg-x11-utils", "xorg-x11-xauth", + "xfce4-terminal", "dbus-x11", "xsetroot", "xset", "xdpyinfo", "xprop", "xorg-x11-xauth", "setxkbmap", "dejavu-sans-fonts"], - "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", - "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xauth", "xorg-setxkbmap", "ttf-dejavu"], + "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", "dbus", + "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xprop", "xorg-xauth", "xorg-setxkbmap", + "ttf-dejavu"], } From 241081ae89343cb9fc27ab3bb71591ebbeb76276 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:10:22 -0700 Subject: [PATCH 30/55] fix(bot-screen): launcher.pid carries the process birth time, not just a pid pid_exists alone trusts a recycled pid: after a reboot or a long-lived gateway, an unrelated process can own the recorded number and status() reports the screen running while stop() SIGTERMs that stranger's whole process group. The pid file now stores " " and _launcher_pid() requires both to match; the pre-identity single-number format, a dead pid and an out-of-range digit string all read as not running (the safe direction: worst case is a spurious start()). (cherry picked from commit 2f9e0e40a764f5a6bb38c02194081dc34c4c8f0b) --- tests/tools/test_bot_desktop_runtime.py | 18 ++++++++++++++++++ tools/bot_desktop/runtime.py | 25 +++++++++++++++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/tests/tools/test_bot_desktop_runtime.py b/tests/tools/test_bot_desktop_runtime.py index 31c28cba9534..caa544f006eb 100644 --- a/tests/tools/test_bot_desktop_runtime.py +++ b/tests/tools/test_bot_desktop_runtime.py @@ -26,6 +26,24 @@ def test_no_running_screen_returns_none_without_grabbing(monkeypatch): assert thumbnail.thumbnail_data_url() is None +def test_recycled_pid_is_not_our_launcher(tmp_path, monkeypatch): + """launcher.pid names pid + create_time; a live pid born at another time is a stranger (recycled pid) + and must read as not running, or stop() would killpg an unrelated session. Legacy single-number + files and absurd digit strings are also not running.""" + import os + + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path) + pidfile = tmp_path / "launcher.pid" + pidfile.write_text(f"{os.getpid()} 12345.0", encoding="utf-8") # alive, wrong birth + assert runtime._launcher_pid() is None + pidfile.write_text(str(os.getpid()), encoding="utf-8") # pre-identity format + assert runtime._launcher_pid() is None + pidfile.write_text("9" * 40 + " 1.0", encoding="utf-8") + assert runtime._launcher_pid() is None + pidfile.write_text(f"{os.getpid()} {runtime._create_time(os.getpid())}", encoding="utf-8") + assert runtime._launcher_pid() == os.getpid() + + def test_recorded_display_held_by_a_live_server_is_not_reused(tmp_path, monkeypatch): """After profile A stops, B may take A's number; A restarting must pick another rather than unlink B's socket and lock.""" diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py index e6ecb8e6a0aa..9f18a3637cc8 100644 --- a/tools/bot_desktop/runtime.py +++ b/tools/bot_desktop/runtime.py @@ -128,12 +128,28 @@ def _pid_alive(pid: int) -> bool: return psutil.pid_exists(pid) +def _create_time(pid: int) -> Optional[float]: + import psutil + try: + return psutil.Process(pid).create_time() + except (psutil.Error, OverflowError, ValueError): + return None + + def _launcher_pid() -> Optional[int]: + """The live launcher's pid, or None. ``launcher.pid`` holds ``" "``: a recycled pid + with a different start time is somebody else's process and must never be reported as ours nor + killed by :func:`stop`. The pre-identity single-number format is treated as not running.""" raw = _read(state_dir() / "launcher.pid") - if not raw or not raw.isdigit(): + pid_s, _, born_s = (raw or "").partition(" ") + if not pid_s.isdigit() or not born_s: + return None + try: + pid, born = int(pid_s), float(born_s) + except ValueError: return None - pid = int(raw) - return pid if _pid_alive(pid) else None + actual = _create_time(pid) + return pid if actual is not None and abs(actual - born) < 0.01 else None def _display_in_use(num: int) -> bool: @@ -295,7 +311,8 @@ def start(*, wait_seconds: float = 15.0) -> DesktopStatus: ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True, close_fds=True) log.close() - (sd / "launcher.pid").write_text(str(proc.pid), encoding="utf-8") + born = _create_time(proc.pid) + (sd / "launcher.pid").write_text(f"{proc.pid} {born if born is not None else 0}", encoding="utf-8") deadline = time.monotonic() + wait_seconds while time.monotonic() < deadline: From a6ed11ca052914832028dc5a6e4b45dce7efd585 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:18:50 -0700 Subject: [PATCH 31/55] fix(bot-screen): hold the display-allocation and a per-profile start lock across spawn _allocate_display released the host-wide flock as soon as it picked a number, but Xvnc writes /tmp/.X-lock only once it is up. Two profiles cold-starting together both picked n; the loser's Xvnc failed and its launcher's stale-lock cleanup could unlink the winner's socket. There was also no per-profile lock at all: two start() calls for one profile spawned two launchers, the second overwrote launcher.pid and the first was orphaned. start() now takes /start.lock (per profile) around the running-check, and the allocation lock around pick + spawn + publish wait (bounded by wait_seconds, default 15s, so a stuck launcher cannot wedge other profiles for long). stop() takes the same per-profile lock so a stop cannot interleave with a start. The launcher does not inherit the lock fds (close_fds=True), so the locks fall with the caller. (cherry picked from commit 1eee1a331e7ac6a8a2860dd7ff2df0150fe12a8b) --- tests/tools/test_bot_desktop_runtime.py | 87 +++++++++++++++++++++++++ tools/bot_desktop/runtime.py | 71 ++++++++++++++------ 2 files changed, 139 insertions(+), 19 deletions(-) diff --git a/tests/tools/test_bot_desktop_runtime.py b/tests/tools/test_bot_desktop_runtime.py index caa544f006eb..fd89a14d6de8 100644 --- a/tests/tools/test_bot_desktop_runtime.py +++ b/tests/tools/test_bot_desktop_runtime.py @@ -2,7 +2,9 @@ from __future__ import annotations +import contextlib import sys +from pathlib import Path import pytest @@ -57,3 +59,88 @@ def test_recorded_display_held_by_a_live_server_is_not_reused(tmp_path, monkeypa assert runtime._allocate_display() != 37 live.clear() assert runtime._allocate_display() == 37, "a free recorded number is reclaimed" + + + +_FAKE_LAUNCHER = """#!/usr/bin/env bash +# Stands in for launcher.sh + Xvnc: the X lock appears only after a delay (the TOCTOU window), then the +# env file + socket are published; stays alive until killed like the real supervisor. +: > "$HERMES_BD_XLOCK_DIR/spawned.$$" +sleep 0.4 +echo $$ > "$HERMES_BD_XLOCK_DIR/.X${HERMES_BD_DISPLAY_NUM}-lock" +: > "$HERMES_BD_SOCKET" +printf 'DISPLAY=:%s\\n' "$HERMES_BD_DISPLAY_NUM" > "$HERMES_BD_ENV_FILE" +sleep 30 +""" + +# One start() per process: state_dir() is HERMES_HOME-scoped and process-global, so two profiles need two +# interpreters — which is also how two gateway profiles race on a real host. +_DRIVER = """ +import json, os, sys +from pathlib import Path +sys.path.insert(0, {repo!r}) +from tools.bot_desktop import runtime +scratch = Path({scratch!r}) +runtime._LAUNCHER = scratch / "launcher.sh" +runtime._X_LOCK_DIR = scratch / "xlocks" +runtime._ALLOC_LOCK = scratch / "alloc.lock" +runtime.missing_binaries = lambda: [] +runtime.geometry = lambda: "800x600" +os.environ["HERMES_BD_XLOCK_DIR"] = str(scratch / "xlocks") +try: + st = runtime.start(wait_seconds=10) + print(json.dumps({{"display": st.display, "pid": st.pid}}), flush=True) +except Exception as exc: + print(json.dumps({{"error": str(exc)}}), flush=True) +sys.stdin.readline() # the test releases us once every driver has reported; we own the launcher, we stop it +runtime.stop() +""" + + +@pytest.fixture +def start_in_fresh_process(tmp_path): + import os + import subprocess + + (tmp_path / "launcher.sh").write_text(_FAKE_LAUNCHER, encoding="utf-8") + (tmp_path / "xlocks").mkdir() + repo = str(Path(__file__).resolve().parents[2]) + procs: list[subprocess.Popen] = [] + + def launch(home: Path) -> subprocess.Popen: + env = {**os.environ, "HERMES_HOME": str(home)} + proc = subprocess.Popen([sys.executable, "-c", _DRIVER.format(repo=repo, scratch=str(tmp_path))], + env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + procs.append(proc) + return proc + + yield launch + for proc in procs: + with contextlib.suppress(OSError): + proc.communicate("go\n", timeout=20) + proc.kill() + + +def _collect(procs): + import json + out = [json.loads(p.stdout.readline()) for p in procs] # every driver holds its launcher until released + assert all("error" not in o for o in out), out + return out + + +@pytest.mark.linux_only +def test_concurrent_cold_starts_of_two_profiles_get_distinct_displays(tmp_path, start_in_fresh_process): + """The allocation lock must outlive the pick: Xvnc writes /tmp/.X-lock well after start() chose n, so + a second profile starting in that window used to pick the same n (and its launcher's stale-lock cleanup + could then unlink the winner's socket).""" + out = _collect([start_in_fresh_process(tmp_path / "a"), start_in_fresh_process(tmp_path / "b")]) + assert len({o["display"] for o in out}) == 2, out + + +@pytest.mark.linux_only +def test_concurrent_starts_of_one_profile_spawn_one_launcher(tmp_path, start_in_fresh_process): + """Two start() calls for one profile spawn ONE launcher; the second used to spawn its own, overwrite + launcher.pid and orphan the first (both callers then reported the last-written pid).""" + out = _collect([start_in_fresh_process(tmp_path / "a"), start_in_fresh_process(tmp_path / "a")]) + assert len({o["pid"] for o in out}) == 1, out + assert len(list((tmp_path / "xlocks").glob("spawned.*"))) == 1 diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py index 9f18a3637cc8..7547eafd7416 100644 --- a/tools/bot_desktop/runtime.py +++ b/tools/bot_desktop/runtime.py @@ -14,6 +14,7 @@ from __future__ import annotations +import contextlib import logging import os import shutil @@ -152,10 +153,13 @@ def _launcher_pid() -> Optional[int]: return pid if actual is not None and abs(actual - born) < 0.01 else None +_X_LOCK_DIR = Path("/tmp") # where X servers write .X-lock (tests point it at a scratch dir) + + def _display_in_use(num: int) -> bool: """A live X server owns ``:num``: its lock file names a running pid. A lock left by a crashed server (dead pid) does not count, so the number can be reclaimed.""" - lock = Path(f"/tmp/.X{num}-lock") + lock = _X_LOCK_DIR / f".X{num}-lock" try: pid = int(lock.read_text(encoding="utf-8").strip()) except (OSError, ValueError): @@ -166,22 +170,34 @@ def _display_in_use(num: int) -> bool: _ALLOC_LOCK = Path("/tmp/.hermes-bot-desktop-alloc.lock") # host-wide: profiles allocate from one band -def _allocate_display() -> int: - """Pick this profile's display number under a host-wide lock. The recorded number is only reused - when no OTHER server holds it now: after profile A stops, B may have taken A's old number, and - A's launcher must never unlink B's socket and lock.""" - import fcntl - with open(_ALLOC_LOCK, "a+", encoding="utf-8") as fh: # windows-footgun: ok — Linux-only runtime +@contextlib.contextmanager +def _flocked(path: Path): + import fcntl # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + with open(path, "a+", encoding="utf-8") as fh: # windows-footgun: ok — Linux-only runtime fcntl.flock(fh.fileno(), fcntl.LOCK_EX) - recorded = _read(state_dir() / "display") - if recorded and recorded.isdigit() and not _display_in_use(int(recorded)): - return int(recorded) - for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1): - if not _display_in_use(num): - return num + try: + yield fh + finally: + fcntl.flock(fh.fileno(), fcntl.LOCK_UN) + + +def _pick_display() -> int: + """Caller holds ``_ALLOC_LOCK``. The recorded number is only reused when no OTHER server holds it now: + after profile A stops, B may have taken A's number, and A's launcher must never unlink B's socket.""" + recorded = _read(state_dir() / "display") + if recorded and recorded.isdigit() and not _display_in_use(int(recorded)): + return int(recorded) + for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1): + if not _display_in_use(num): + return num raise RuntimeError("no free X display number in the Bot Desktop band") +def _allocate_display() -> int: + with _flocked(_ALLOC_LOCK): + return _pick_display() + + def desktop_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: """``base_env`` (default ``os.environ``) with this profile's DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS merged in when its desktop is running. Unchanged otherwise, so hosts with a real seat keep it. @@ -273,20 +289,29 @@ def _profile_name() -> str: def start(*, wait_seconds: float = 15.0) -> DesktopStatus: """Start this profile's desktop (idempotent). Blocks until the launcher publishes its env file or - ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker.""" + ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker. + + Two locks, both held from the running-check to the launcher's publish: the per-profile ``start.lock`` + so two start() calls for one profile spawn one launcher (the loser sees it running), and the host-wide + display-allocation lock so a second profile cannot pick the same number before this Xvnc has written + ``/tmp/.X-lock`` (it would then fail and its launcher's stale-lock cleanup could remove our socket).""" if not is_supported_host(): raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") missing = missing_binaries() if missing: hint = install_command() or "install TigerVNC (Xvnc) and the Xfce core components" raise RuntimeError(f"Bot Desktop needs {', '.join(missing)} on the gateway host. Install: {hint}") - if _launcher_pid() is not None and published_env().get("DISPLAY"): - return status() - sd = state_dir() sd.mkdir(parents=True, exist_ok=True) os.chmod(sd, 0o700) - num = _allocate_display() + with _flocked(sd / "start.lock"): + if _launcher_pid() is not None and published_env().get("DISPLAY"): + return status() + with _flocked(_ALLOC_LOCK): + return _spawn_and_wait(sd, _pick_display(), wait_seconds) + + +def _spawn_and_wait(sd: Path, num: int, wait_seconds: float) -> DesktopStatus: (sd / "display").write_text(str(num), encoding="utf-8") env_file = sd / "env" env_file.unlink(missing_ok=True) @@ -328,8 +353,16 @@ def start(*, wait_seconds: float = 15.0) -> DesktopStatus: def stop() -> bool: """Stop this profile's desktop; True when a running launcher was signalled.""" - pid = _launcher_pid() + if not is_supported_host(): + return False sd = state_dir() + sd.mkdir(parents=True, exist_ok=True) + with _flocked(sd / "start.lock"): + return _stop_locked(sd) + + +def _stop_locked(sd: Path) -> bool: + pid = _launcher_pid() if pid is None: (sd / "env").unlink(missing_ok=True) return False From 0b7e9df1965dc92b1d0951bbc5e6bec9bee3d8d7 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:18:50 -0700 Subject: [PATCH 32/55] fix(bot-screen): truncate launcher.log on each start The log was opened in append mode and nothing rotated it; a long-lived gateway restarting its screen grew it without bound and the tail start() reports on failure mixed launches. Each start now owns the file. (cherry picked from commit ea1e3ac737b3d0b76afc5ec5373c5d3248207820) --- tools/bot_desktop/runtime.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py index 7547eafd7416..16e4569bfb23 100644 --- a/tools/bot_desktop/runtime.py +++ b/tools/bot_desktop/runtime.py @@ -331,7 +331,8 @@ def _spawn_and_wait(sd: Path, num: int, wait_seconds: float) -> DesktopStatus: if (browser := dock_launch()) is not None: # first-run / default-browser dialogs would sit between the human and the bot's tabs child_env["HERMES_BD_BROWSER_EXEC"] = f"{browser[0]} --user-data-dir={browser[1]} --no-first-run --no-default-browser-check" - log = open(sd / "launcher.log", "ab") # noqa: SIM115 — handed to the child, closed by it + # Truncated per start: the log is a diagnostic for THIS launch, and nothing rotates it otherwise. + log = open(sd / "launcher.log", "wb") # noqa: SIM115 — handed to the child, closed by it proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host) ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True, close_fds=True) From eaece2e034ec33e10aefbed5230295c9d297d5a4 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:21:21 -0700 Subject: [PATCH 33/55] fix(bot-screen): launcher stays the supervisor so a dead Xfce session takes Xvnc with it `exec dbus-run-session ...` replaced the launcher process, discarding the `trap 'kill $XVNC_PID' EXIT`. When xfce4-panel (the session's foreground process) exited on its own, dbus-run-session ended, launcher.pid pointed at nothing and Xvnc lived on as an orphan that runtime.stop() could no longer find (live-reproduced: panel killed -> launcher gone, Xvnc :20 still serving). Without exec the script waits for the session, reaps Xvnc and exits with the session's status; the process group runtime.stop() signals is unchanged (launcher is still the session leader), verified live: 16 group members before stop, none after, no /tmp/.X20-lock left. (cherry picked from commit 6e975f900f06ed2e85b2ffe0e462378ea6dd2e83) --- tools/bot_desktop/launcher.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh index 75006e0193b6..842a2582f53b 100755 --- a/tools/bot_desktop/launcher.sh +++ b/tools/bot_desktop/launcher.sh @@ -245,7 +245,9 @@ xset -display "$DISPLAY" s off -dpms s noblank 2>/dev/null || true # dbus-run-session scopes the bus to this subshell: no leaked dbus-daemons on restart. The env file # is written from INSIDE the bus so DBUS_SESSION_BUS_ADDRESS is the real one; runtime.py and every # cua-driver / browser spawn for this profile source it. -exec dbus-run-session -- bash -c ' +# Not exec'd: this script stays the supervisor so the EXIT trap above still reaps Xvnc when the Xfce +# session dies on its own (exec would replace the trap's owner and orphan the X server). +dbus-run-session -- bash -c ' set -e umask 077 printf "DISPLAY=%s\nXAUTHORITY=%s\nDBUS_SESSION_BUS_ADDRESS=%s\nXDG_CONFIG_HOME=%s\nXDG_CACHE_HOME=%s\nXDG_DATA_HOME=%s\n" \ @@ -256,4 +258,6 @@ exec dbus-run-session -- bash -c ' for _ in $(seq 1 50); do xprop -root _NET_SUPPORTING_WM_CHECK >/dev/null 2>&1 && break; sleep 0.1; done xfdesktop --sm-client-disable --disable-wm-check & exec xfce4-panel --sm-client-disable --disable-wm-check -' +' && rc=0 || rc=$? +kill "$XVNC_PID" 2>/dev/null || true +exit "$rc" From a456aa0ad015b3d029105cb87f4b414cd745a25e Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:24:39 -0700 Subject: [PATCH 34/55] fix(bot-screen): install timeout kills the package manager's group; atomic slot claim The timeout Timer called proc.kill, which only reaches sudo: apt/dnf kept running as root in the new session, holding the dpkg lock, while the per-profile install slot was released and a retry would collide with it. The timer now SIGKILLs the whole process group start_new_session created. install.claim() atomically takes the slot (raises InstallBusy when held) and install_packages(claimed=True) skips re-claiming but still releases, so the gateway handler can claim before spawning its worker thread instead of a read-only assert_not_running that two Install clicks both pass. (cherry picked from commit 31cce2b634fc6b84dd6422abec4ecee7b770af46) --- tests/tools/test_bot_desktop_install.py | 52 +++++++++++++++++++++++++ tools/bot_desktop/install.py | 47 ++++++++++++++++------ 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/tests/tools/test_bot_desktop_install.py b/tests/tools/test_bot_desktop_install.py index 5b0912d7e827..a3e5392a66a2 100644 --- a/tests/tools/test_bot_desktop_install.py +++ b/tests/tools/test_bot_desktop_install.py @@ -45,3 +45,55 @@ def slow_run(cmd, *, ask_password, on_line, timeout_seconds): gate.set() worker.join(5) install.assert_not_running() # slot released once the run finishes + + +def test_claim_is_atomic_and_refuses_a_second_claim(): + """The gateway claims BEFORE spawning its worker; a second Install click must fail at claim time, not + pass a read-only check and race the worker for the slot.""" + key = install.claim() + with pytest.raises(install.InstallBusy): + install.claim() + with pytest.raises(install.InstallBusy): + install.install_packages(ask_password=lambda: "pw", on_line=lambda _l: None) + install.release(key) + install.claim() # free again + install.release(key) + + +@pytest.mark.linux_only +def test_timeout_kills_the_package_managers_whole_process_group(monkeypatch): + """sudo forks the package manager into the same (new) session; killing sudo alone leaves apt/dnf + holding the dpkg lock as root. The timeout must take the group.""" + import subprocess + import time + + monkeypatch.setattr(install, "_sudo_nopasswd", lambda: True) + # stand-in for `sudo apt-get ...`: a parent that spawns a child and waits, both in the new session + fake = ["sudo"] + real_popen = subprocess.Popen + + def popen(argv, **kw): + if argv[:1] != fake: + return real_popen(argv, **kw) + return real_popen(["bash", "-c", "sleep 30 >/dev/null 2>&1 & echo child $!; wait"], **kw) + + monkeypatch.setattr(install.subprocess, "Popen", popen) + lines: list[str] = [] + code = install._run("sudo apt-get install -y x", ask_password=lambda: "", on_line=lines.append, timeout_seconds=0.5) + assert code != 0 + child = next(int(line.split()[1]) for line in lines if line.startswith("child ")) + from pathlib import Path + + def gone() -> bool: # /proc-based: a reparented orphan sits outside our subtree, where os.kill(pid, 0) is guarded + try: + return "Z" in (Path(f"/proc/{child}/stat").read_text().rsplit(")", 1)[1].split() or ["Z"])[0] + except OSError: + return True + + for _ in range(50): # the child must die with the group, not linger reparented to init + if gone(): + break + time.sleep(0.1) + else: + subprocess.run(["kill", "-9", str(child)], check=False) + pytest.fail("grandchild survived the install timeout") diff --git a/tools/bot_desktop/install.py b/tools/bot_desktop/install.py index ef374a62e3ec..ee44b7e06b5c 100644 --- a/tools/bot_desktop/install.py +++ b/tools/bot_desktop/install.py @@ -12,9 +12,11 @@ from __future__ import annotations +import contextlib import logging import os import shlex +import signal import subprocess import threading from typing import Callable, Optional @@ -38,24 +40,41 @@ def assert_not_running() -> None: raise InstallBusy("an install is already running for this profile") -def install_packages(*, ask_password: Callable[[], str], on_line: Callable[[str], None], - timeout_seconds: float = 900.0) -> int: - """Run the package install; returns the process exit code (0 = success, ``-1`` = cancelled).""" - if not runtime.is_supported_host(): - raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") - cmd = runtime.install_command() - if cmd is None: - raise RuntimeError("no supported package manager (apt-get, dnf, pacman) found on this host") +def claim() -> str: + """Atomically take this profile's install slot; raises :class:`InstallBusy` when taken. A caller that + claims before handing off to a worker passes ``claimed=True`` to :func:`install_packages`, which then + owns releasing it — a check-then-spawn pair (``assert_not_running`` + later claim on the worker) lets + two Install clicks both pass the check.""" key = hermes_home_key() with _install_lock: if key in _running: raise InstallBusy("an install is already running for this profile") _running.add(key) + return key + + +def release(key: str) -> None: + with _install_lock: + _running.discard(key) + + +def install_packages(*, ask_password: Callable[[], str], on_line: Callable[[str], None], + timeout_seconds: float = 900.0, claimed: bool = False) -> int: + """Run the package install; returns the process exit code (0 = success, ``-1`` = cancelled). + ``claimed=True``: the caller already holds the slot via :func:`claim`; it is released here either way.""" + key = hermes_home_key() if claimed else None try: + if not runtime.is_supported_host(): + raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") + cmd = runtime.install_command() + if cmd is None: + raise RuntimeError("no supported package manager (apt-get, dnf, pacman) found on this host") + if key is None: + key = claim() return _run(cmd, ask_password=ask_password, on_line=on_line, timeout_seconds=timeout_seconds) finally: - with _install_lock: - _running.discard(key) + if key is not None: + release(key) def _sudo_nopasswd() -> bool: @@ -90,7 +109,13 @@ def _run(cmd: str, *, ask_password: Callable[[], str], on_line: Callable[[str], proc.stdin.close() # type: ignore[union-attr] except OSError: pass - timer = threading.Timer(timeout_seconds, proc.kill) + # The package manager runs in its own session (start_new_session); killing only sudo would leave apt/dnf + # running as root with the dpkg lock while the slot is released, so the whole group goes. + def _kill_group() -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) # windows-footgun: ok — Linux-only (is_supported_host) + + timer = threading.Timer(timeout_seconds, _kill_group) timer.start() try: for line in proc.stdout: # type: ignore[union-attr] From b3eb315b9430f08979a93570bc07d1832bb51265 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:25:17 -0700 Subject: [PATCH 35/55] fix(bot-screen): CLI `screen install` runs through the shared installer `hermes computer-use screen install` was a second installer: it ran the package command with shell=True and bypassed install._running, so a CLI install and a Desktop-pane Install could run the same apt transaction concurrently. It now calls install.install_packages with a getpass password callback and print as the line sink, sharing the per-profile slot, the list-form spawn and the stdin (-S) sudo hand-off. -y keeps its meaning (skip the confirmation prompt). (cherry picked from commit 45f58eb19a0e72ee01bae8bd101d28dd59b5b451) --- hermes_cli/subcommands/computer_use_screen.py | 14 ++++++--- .../test_computer_use_screen_install.py | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 tests/hermes_cli/test_computer_use_screen_install.py diff --git a/hermes_cli/subcommands/computer_use_screen.py b/hermes_cli/subcommands/computer_use_screen.py index 236c6c3f5a08..5e76b521f991 100644 --- a/hermes_cli/subcommands/computer_use_screen.py +++ b/hermes_cli/subcommands/computer_use_screen.py @@ -4,8 +4,8 @@ from __future__ import annotations +import getpass import json -import subprocess import sys @@ -54,7 +54,7 @@ def _screen_stop(args) -> int: def _screen_install(args) -> int: - from tools.bot_desktop import runtime + from tools.bot_desktop import install, runtime if not runtime.is_supported_host(): print("Bot Desktop screens run on Linux gateway hosts only.") return 1 @@ -71,10 +71,16 @@ def _screen_install(args) -> int: answer = input("Proceed? [Y/n] ").strip().lower() if answer not in ("", "y", "yes"): return 1 - rc = subprocess.run(cmd, shell=True, stdin=None).returncode # windows-footgun: ok — Linux-only, apt/dnf/pacman + # Same runner as the Desktop pane's Install button: one per-profile slot, list-form spawn, sudo password + # via stdin (-S) and never on the command line. + try: + rc = install.install_packages(ask_password=lambda: getpass.getpass("[sudo] password: "), on_line=print) + except install.InstallBusy as exc: + print(f"Bot Desktop: {exc}") + return 1 if rc != 0: print(f"Bot Desktop: installer exited {rc}") - return rc + return rc or 1 missing = runtime.missing_binaries() if missing: print("Bot Desktop: still missing " + ", ".join(missing)) diff --git a/tests/hermes_cli/test_computer_use_screen_install.py b/tests/hermes_cli/test_computer_use_screen_install.py new file mode 100644 index 000000000000..a0199600801c --- /dev/null +++ b/tests/hermes_cli/test_computer_use_screen_install.py @@ -0,0 +1,29 @@ +"""`hermes computer-use screen install` shares the Desktop pane's installer (one lock, list-form spawn).""" + +from __future__ import annotations + +import argparse +import subprocess + +from hermes_cli.subcommands.computer_use_screen import build_screen_parser +from tools.bot_desktop import install, runtime + + +def test_cli_install_goes_through_the_shared_installer(monkeypatch): + monkeypatch.setattr(runtime, "is_supported_host", lambda: True) + monkeypatch.setattr(runtime, "missing_binaries", lambda: ["Xvnc"]) + monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y tigervnc-standalone-server") + monkeypatch.setattr(subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(AssertionError("shell install spawned"))) + calls = [] + + def fake_install(*, ask_password, on_line, **kw): + calls.append((callable(ask_password), callable(on_line))) + return 0 + + monkeypatch.setattr(install, "install_packages", fake_install) + monkeypatch.setattr(runtime, "missing_binaries", lambda: ["Xvnc"] if not calls else []) + parser = argparse.ArgumentParser() + build_screen_parser(parser.add_subparsers(), lambda sub, help_text: sub.add_argument("--json", action="store_true")) + args = parser.parse_args(["screen", "install", "-y"]) + assert args.screen_func(args) == 0 + assert calls == [(True, True)] From dcc8735ab246ce4488106e3c26d034d121afabff Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:25:43 -0700 Subject: [PATCH 36/55] feat(computer-use): display identity helpers for cached-backend rebind _get_backend keys its cache on permission mode only, so a cua-driver cached before the Bot Desktop started (or restarted on another display number) keeps acting on the old seat, or on no display. desktop_identity() returns the DISPLAY a fresh spawn would get and backend_display_stale() compares it with the identity recorded at cache time; tool.py (owned by another lane) records the identity next to _backend_permission_modes[sid] and detaches+stops a stale backend the way the /yolo mode swap does. (cherry picked from commit 896304bcbb98b12f821c6e19f10b9f174fa08889) --- tests/tools/test_computer_use_backend_rebind.py | 13 +++++++++++++ tools/computer_use/cua_backend.py | 12 ++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/tools/test_computer_use_backend_rebind.py diff --git a/tests/tools/test_computer_use_backend_rebind.py b/tests/tools/test_computer_use_backend_rebind.py new file mode 100644 index 000000000000..c2ca5b99071a --- /dev/null +++ b/tests/tools/test_computer_use_backend_rebind.py @@ -0,0 +1,13 @@ +"""A cached computer_use backend is bound to the display it spawned on; the identity helpers notice a change.""" + +from __future__ import annotations + +from tools.computer_use import cua_backend + + +def test_backend_display_identity_tracks_the_display_a_spawn_would_get(): + before = cua_backend.desktop_identity({"HOME": "/x"}) # no screen yet + after = cua_backend.desktop_identity({"HOME": "/x", "DISPLAY": ":37"}) # Bot Desktop came up + assert before == "" and after == ":37" + assert cua_backend.backend_display_stale(before, after) + assert not cua_backend.backend_display_stale(after, cua_backend.desktop_identity({"DISPLAY": ":37"})) diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index 812d75560927..d6075d2e7aab 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -100,6 +100,18 @@ def _computer_use_max_image_dimension() -> Optional[int]: dim = 1456 return dim if dim > 0 else None +def desktop_identity(env: Optional[Dict[str, str]] = None) -> str: + """The screen a backend spawned from ``env`` acts on: its DISPLAY (``''`` when none). Recorded next to the + cached backend so a Bot Desktop that starts (or restarts on another number) AFTER the backend was cached is + noticed — the cached cua-driver still points at the old seat or at no display at all.""" + return str((cua_driver_child_env(env) if env is None else env).get("DISPLAY") or "") + + +def backend_display_stale(recorded: str, current: str) -> bool: + """True when a cached backend's recorded display identity no longer matches the one a fresh spawn would get.""" + return (recorded or "") != (current or "") + + def cua_driver_child_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: """Env for spawning cua-driver: ``base_env`` (default ``os.environ``) plus ``CUA_DRIVER_RS_TELEMETRY_ENABLED=0`` unless the user opted in, plus the native-Wayland bridge (``computer_use.native_wayland`` config opt-in, only when From 64524cce54416ef483ebf0b5105d0a98d8cb9706 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:10:48 -0700 Subject: [PATCH 37/55] fix(bot-screen): cross-process lease changes reach Desktop clients as display.lease The lease is changed by whatever process hosts the agent (messaging gateway request_handoff, hermes chat takeover, cron worker release), but lease.on_change fires only inside the writing process, so the serve backend's in-process listener never saw those moves: the Desktop's "Bot needs you" badge, hero tone and pane state stayed stale until the pane was reopened. Add methods_display_watch: one daemon thread (started on the first Desktop WebSocket peer, next to the skin/change watcher) stats /bot-desktop/lease.json for the launch home and every served profile home every 0.5s, re-reads the lease only when the mtime moved, and broadcasts the same display.lease payload the in-process listener sends when the epoch changed. In-process transitions record their epoch via lease.on_change so the file move they cause is not re-broadcast; first sighting seeds silently so a boot never fires a stale event. Two-process tests (subprocess with its own HERMES_HOME drives request_handoff / release) fail on base with AttributeError and pass here; live-verified against `hermes serve --port 9891`: the event arrived 0.42s / 0.50s after the cross-process transition. (cherry picked from commit 16b30a099b7a1450ce3841aa73e9d7b06c01f713) --- tests/tui_gateway/test_display_watch.py | 80 +++++++++++++++++++++++ tui_gateway/methods_display_watch.py | 86 +++++++++++++++++++++++++ tui_gateway/server.py | 5 +- tui_gateway/ws.py | 1 + 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 tests/tui_gateway/test_display_watch.py create mode 100644 tui_gateway/methods_display_watch.py diff --git a/tests/tui_gateway/test_display_watch.py b/tests/tui_gateway/test_display_watch.py new file mode 100644 index 000000000000..b9248724c9d2 --- /dev/null +++ b/tests/tui_gateway/test_display_watch.py @@ -0,0 +1,80 @@ +"""Lease transitions made by ANOTHER process reach `hermes serve` clients as ``display.lease``. + +The agent lives in whatever process hosts it (messaging gateway, ``hermes chat``, a cron worker); +``lease.on_change`` is in-process only, so the serve backend must notice the file move itself. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] + + +def _other_process(home: Path, stmt: str) -> None: + env = {**os.environ, "HERMES_HOME": str(home)} + subprocess.run( # noqa: S603 + [sys.executable, "-c", f"import sys; sys.path.insert(0, {str(REPO)!r}); " + f"from tools.bot_desktop import lease; {stmt}"], + cwd=str(REPO), env=env, check=True, timeout=60, stdin=subprocess.DEVNULL) + + +def _wait_for(pred, timeout: float = 3.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(0.05) + return pred() + + +def _watching(server, home: Path, monkeypatch) -> list: + from hermes_constants import hermes_home_key + events: list = [] + monkeypatch.setattr(server, "_broadcast_global_event", lambda ev, payload=None: events.append((ev, payload))) + monkeypatch.setattr(server, "_hermes_home", str(home)) + server._ensure_lease_watcher() + assert _wait_for(lambda: hermes_home_key(home) in server._lease_epochs), "watcher never seeded the home" + return events + + +def _lease_events(events, **want): + return [p for ev, p in events if ev == "display.lease" and all(p["lease"].get(k) == v for k, v in want.items())] + + +def test_handoff_requested_in_another_process_is_broadcast(tmp_path, monkeypatch): + import tui_gateway.server as server + from hermes_constants import hermes_home_key + home = tmp_path / "home" + home.mkdir() + events = _watching(server, home, monkeypatch) + + _other_process(home, 'lease.request_handoff("probe")') + + assert _wait_for(lambda: _lease_events(events, pending_handoff="probe")), events + (payload,) = _lease_events(events, pending_handoff="probe") + assert payload["profile_key"] == hermes_home_key(home) + + +def test_release_in_another_process_is_broadcast_and_local_transition_not_duplicated(tmp_path, monkeypatch): + import tui_gateway.server as server + from tools.bot_desktop import lease + home = tmp_path / "home" + home.mkdir() + events = _watching(server, home, monkeypatch) + server._install_lease_listener() # what display.status does: in-process transitions broadcast here + + lease.acquire("viewer-1", profile_key=str(home)) + assert _wait_for(lambda: _lease_events(events, holder="human")) + before = len(events) + server._poll_lease_files() # the file moved too, but the watcher saw that epoch locally: no re-broadcast + assert len(events) == before + + _other_process(home, 'lease.release("viewer-1")') + + assert _wait_for(lambda: _lease_events(events, holder="agent")), events + assert len(_lease_events(events, holder="agent")) == 1 diff --git a/tui_gateway/methods_display_watch.py b/tui_gateway/methods_display_watch.py new file mode 100644 index 000000000000..7c0efb162210 --- /dev/null +++ b/tui_gateway/methods_display_watch.py @@ -0,0 +1,86 @@ +"""Cross-process lease watcher: ``display.lease`` for transitions made OUTSIDE ``hermes serve``. + +The lease (``tools.bot_desktop.lease``) lives on disk and is changed by whatever process hosts the +agent — the messaging gateway's ``request_handoff``, a ``hermes chat`` takeover, a cron worker's +release. ``lease.on_change`` only fires in the writing process, so ``methods_display``'s in-process +listener never sees those; the Desktop's "Bot needs you" badge and hero tone stayed stale until the +pane was reopened. One daemon thread stats every served home's ``bot-desktop/lease.json`` (launch +home + ``_served_profile_homes``) every 0.5s and broadcasts the SAME ``display.lease`` payload when +the epoch moves. Bodies are rebound onto server.py's globals (method_ctx.bind_module). +""" + +from __future__ import annotations + +import threading +import time +from pathlib import Path + +from .method_ctx import bind_module + +_LEASE_POLL_S = 0.5 +_lease_watcher_started = threading.Event() +# profile key → last epoch broadcast or seen (in-process transitions record theirs too, so a change +# made by THIS process is not re-broadcast when its file write is noticed a tick later). +_lease_epochs: dict[str, int] = {} +_lease_mtimes: dict[str, int | None] = {} + + +def _lease_event_payload(profile_key: str, lease) -> dict: + # Mirrors methods_display._install_lease_listener's nested payload; keep the two in step. + return {"profile_key": profile_key, "lease": lease.as_dict()} + + +def _watched_lease_homes() -> list[Path]: + return [Path(_hermes_home), *_served_profile_homes] + + +def _poll_lease_files() -> None: + """One pass: read a home's lease only when its file mtime moved; broadcast when the epoch did.""" + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _bd_lease + for home in _watched_lease_homes(): + key = hermes_home_key(home) + try: + mtime = (home / "bot-desktop" / "lease.json").stat().st_mtime_ns + except OSError: + mtime = None + if mtime == _lease_mtimes.get(key, 0): + continue + _lease_mtimes[key] = mtime + lease = _bd_lease.get(str(home)) + if key not in _lease_epochs: # first sighting seeds silently: display.status carries it + _lease_epochs[key] = lease.epoch + continue + if lease.epoch == _lease_epochs[key]: + continue + _lease_epochs[key] = lease.epoch + _broadcast_global_event("display.lease", _lease_event_payload(key, lease)) + + +def _ensure_lease_watcher() -> None: + """Once per process, from the first display.* call: start the lease-file poll thread and mark + in-process transitions as seen so they broadcast exactly once (via the in-process listener).""" + if _lease_watcher_started.is_set(): + return + _lease_watcher_started.set() + from tools.bot_desktop import lease as _bd_lease + + def _seen_locally(profile_key: str, lease) -> None: + # methods_display's listener broadcasts in-process transitions; only then is the file + # move ours to skip. Before display.status installed it, the poll below carries them. + if _lease_listener_installed.is_set(): + _lease_epochs[profile_key] = lease.epoch + _bd_lease.on_change(_seen_locally) + + def _loop() -> None: + while True: + try: + _poll_lease_files() + except Exception: # noqa: BLE001 - a torn read must not kill the watcher + logger.debug("lease watcher poll failed", exc_info=True) + time.sleep(_LEASE_POLL_S) + threading.Thread(target=_loop, name="hermes-lease-watcher", daemon=True).start() + + +def register(server) -> None: + bind_module(globals(), server, skip=("_",)) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index dcc55c923f3f..b5101afc0ba2 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -3230,7 +3230,8 @@ def _resolve_name(name: str) -> str: methods_projects as _methods_projects, methods_session_foreign as _methods_session_foreign, methods_session_control as _methods_session_control, methods_subagents as _methods_subagents, methods_vault as _methods_vault, methods_free_tier as _methods_free_tier, - methods_connectors as _methods_connectors, methods_display as _methods_display) + methods_connectors as _methods_connectors, methods_display as _methods_display, + methods_display_watch as _methods_display_watch) for _m in ( _session_transports, _session_reaper, _session_lifecycle, _session_workdir, _compute_host_bridge, _model_switch, @@ -3241,6 +3242,6 @@ def _resolve_name(name: str) -> str: _methods_config_set, _methods_complete, _methods_tools, _methods_profiles, _methods_images, _methods_bot_relay, _prompt_turn, _billing_view, _methods_projects, _methods_session_foreign, _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors, - _methods_display): + _methods_display, _methods_display_watch): _m.register(sys.modules[__name__]) del _m diff --git a/tui_gateway/ws.py b/tui_gateway/ws.py index 480ed3663694..448224fab1a8 100644 --- a/tui_gateway/ws.py +++ b/tui_gateway/ws.py @@ -304,6 +304,7 @@ def _error(code: int, message: str, req_id: Any) -> dict: # Live-apply skins Hermes activates mid-conversation, and track this peer for session-less # global broadcasts write_json can't route. server._ensure_skin_watcher() + server._ensure_lease_watcher() # cross-process lease moves → display.lease server.register_live_transport(transport) # Cross-backend liveness: a heartbeat row lets the startup orphan sweep tell "live but idle # backend" from "truly orphaned". Idempotent and once-per-process, like the orphan sweep (the From 31bbb7b6a3779cb29f3fb37f02cf800c1a650e0c Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:29:45 -0700 Subject: [PATCH 38/55] fix(bot-screen): agent attaches to a human-started dock Browser instead of dying on the profile singleton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dock's Browser launched raw Chromium on the shared user-data-dir with no automation endpoint. When the human opened it first and handed back, agent-browser's own launch was forwarded into their instance by Chromium's ProcessSingleton and exited 21 without a DevToolsActivePort, so every browser_navigate failed until the human closed their window. The reverse order worked, which is why it slipped through. - The dock command carries --remote-debugging-port=0, so a human-started instance advertises a port in /DevToolsActivePort (browser.dock_command; runtime.start reads it). - browser.running_instance_cdp_port() trusts that file only when SingletonLock's pid is alive AND the port accepts a connection (both files outlive a closed Chromium), and never for the instance the calling agent-browser session launched itself: handing that daemon --cdp makes it treat the launch as a config change, close its browser and attach to the port that just died with it (seen live). - The local argv builder appends --cdp to the --session launch when such an instance exists, so the same daemon (and its snapshot refs) drives the human's window. Live, HERMES_HOME=/tmp/bs-f-home on this host: human-first — dock instance up, navigate x2 succeeded, one Chromium main process (same pid) throughout; agent-first — navigate, dock click, navigate x2 succeeded, one process throughout. Before the fix human-first returned "Chrome exited early (exit code: 21) ... Failed to create SingletonLock". (cherry picked from commit d732fad0af005ffd38eca153dd29c0f7e9dd6bc7) --- tests/tools/test_bot_desktop_browser.py | 67 ++++++++++++++++++++++++- tools/bot_desktop/browser.py | 65 +++++++++++++++++++++++- tools/bot_desktop/runtime.py | 5 +- tools/browser_tool_session.py | 15 ++++++ 4 files changed, 147 insertions(+), 5 deletions(-) diff --git a/tests/tools/test_bot_desktop_browser.py b/tests/tools/test_bot_desktop_browser.py index b3bb6ce9e806..2cfdc5523b47 100644 --- a/tests/tools/test_bot_desktop_browser.py +++ b/tests/tools/test_bot_desktop_browser.py @@ -1,7 +1,12 @@ -"""The dock's Browser and agent-browser resolve to one identity: same executable, same user-data-dir.""" +"""The dock's Browser and agent-browser resolve to one identity: same executable, same user-data-dir — +and when a human opened that browser first, the agent attaches to it instead of launching a second one +(Chromium's profile singleton would forward the launch and kill it without a DevTools endpoint).""" from __future__ import annotations +import os +import socket + from tools.bot_desktop import browser, runtime @@ -24,3 +29,63 @@ def test_user_pinned_profile_wins(tmp_path, monkeypatch): monkeypatch.setenv("AGENT_BROWSER_PROFILE", str(tmp_path / "mine")) monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bot-desktop") assert browser.profile_dir() == tmp_path / "mine" + + +def test_dock_browser_advertises_a_devtools_port(): + """A human-started instance must be attachable, or the agent can never drive it afterwards.""" + assert "--remote-debugging-port=" in browser.dock_command("/opt/chrome", "/p/dir").split()[2] + + +def _fake_running_instance(user_data_dir, pid: int, port: int) -> None: + (user_data_dir / "DevToolsActivePort").write_text(f"{port}\n/devtools/browser/abc\n", encoding="utf-8") + os.symlink(f"host-{pid}", user_data_dir / "SingletonLock") + + +def test_running_instance_port_requires_live_pid_and_open_port(tmp_path): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + port = listener.getsockname()[1] + try: + _fake_running_instance(tmp_path, os.getpid(), port) + assert browser.running_instance_cdp_port(str(tmp_path)) == port + + # Both files outlive a closed Chromium: a dead pid must not be trusted. + os.unlink(tmp_path / "SingletonLock") + os.symlink("host-2147483000", tmp_path / "SingletonLock") + assert browser.running_instance_cdp_port(str(tmp_path)) is None + finally: + listener.close() + # Live pid, port no longer accepting: still not attachable. + os.unlink(tmp_path / "SingletonLock") + os.symlink(f"host-{os.getpid()}", tmp_path / "SingletonLock") + assert browser.running_instance_cdp_port(str(tmp_path)) is None + assert browser.running_instance_cdp_port(str(tmp_path / "missing")) is None + + +def test_agent_attaches_to_human_started_browser(monkeypatch): + """With a live dock instance on the shared profile the local argv carries ``--cdp ``; without one + it stays a plain ``--session`` launch.""" + from tools import browser_tool_session as session + + monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":37"}) + monkeypatch.setattr(session._cloud, "_get_browser_engine", lambda: "auto") + monkeypatch.setattr(session._cloud, "_is_headed_mode", lambda: False) + monkeypatch.setattr(session, "_agent_browser_argv", lambda cmd: [cmd]) + argvs: list = [] + + def spawn(task_id, session_info, cmd_parts, *rest): + argvs.append(cmd_parts) + return {"success": True} + + monkeypatch.setattr(session, "_spawn_and_collect", spawn) + monkeypatch.setattr(session._lp, "_lightpanda_fallback_reason", lambda *a: None) + info = {"session_name": "h_abc", "cdp_url": None} + + monkeypatch.setattr(browser, "running_instance_cdp_port", lambda d, **kw: 41234) + session._run_browser_command_unfenced("t", "open", ["https://x"], 10, None, "agent-browser", info) + assert argvs[-1][:5] == ["agent-browser", "--session", "h_abc", "--cdp", "41234"] + + monkeypatch.setattr(browser, "running_instance_cdp_port", lambda d, **kw: None) + session._run_browser_command_unfenced("t", "open", ["https://x"], 10, None, "agent-browser", info) + assert "--cdp" not in argvs[-1] and argvs[-1][:3] == ["agent-browser", "--session", "h_abc"] diff --git a/tools/bot_desktop/browser.py b/tools/bot_desktop/browser.py index a3fd382e0796..08d7489ee3d8 100644 --- a/tools/bot_desktop/browser.py +++ b/tools/bot_desktop/browser.py @@ -3,7 +3,10 @@ The agent drives Chromium through agent-browser; a human who takes over clicks the dock's Browser icon. Both must be THE SAME browser — same binary, same ``--user-data-dir`` — or the human logs in to a jar the bot never sees. Chromium's singleton makes a second launch on the same user-data-dir -open a window in the running instance, which is exactly the hand-over we want. +open a window in the running instance, which is exactly the hand-over we want — in ONE direction. When the +human's dock instance is already up, agent-browser's own launch is forwarded to it and dies without a +DevTools endpoint, so the dock exposes a debugging port and the agent ATTACHES to it (see +:func:`running_instance_cdp_port`) instead of launching. """ from __future__ import annotations @@ -11,6 +14,7 @@ import glob import os import shutil +import socket from pathlib import Path from typing import Optional, Tuple @@ -49,6 +53,65 @@ def dock_launch() -> Optional[Tuple[str, str]]: return (exe, str(profile_dir())) if exe else None +def dock_command(exe: str, user_data_dir: str) -> str: + """Shell line the dock's Browser icon runs. ``--remote-debugging-port=0`` makes a human-started + instance attachable (Chromium writes the chosen port to ``/DevToolsActivePort``); + first-run / default-browser dialogs would sit between the human and the bot's tabs.""" + return f"{exe} --user-data-dir={user_data_dir} --remote-debugging-port=0 --no-first-run --no-default-browser-check" + + +def running_instance_cdp_port(user_data_dir: str, *, exclude_session: Optional[str] = None) -> Optional[int]: + """DevTools port of a Chromium currently running on ``user_data_dir``, or ``None``. + + Both files outlive a crashed or closed Chromium: ``SingletonLock`` is a symlink to ``host-pid`` and + ``DevToolsActivePort`` keeps the last port, so the pid must be alive AND the port must accept a + connection before it is trusted. An instance agent-browser launched for ``exclude_session`` itself is + reported as ``None``: its daemon already owns that browser, and handing it ``--cdp`` would make it + close the browser as a config change and then attach to the port that just died with it. + """ + try: + with open(os.path.join(user_data_dir, "DevToolsActivePort"), encoding="utf-8") as fh: + port_line = fh.readline().strip() + target = os.readlink(os.path.join(user_data_dir, "SingletonLock")) + except OSError: + return None + _host, _, pid_text = target.rpartition("-") + if not (port_line.isdigit() and pid_text.isdigit()) or not _pid_alive(int(pid_text)): + return None + if exclude_session and _launched_by_session(int(pid_text)) == exclude_session: + return None + port = int(port_line) + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + pass + except OSError: + return None + return port + + +def _launched_by_session(chromium_pid: int) -> Optional[str]: + """``AGENT_BROWSER_SESSION`` of the agent-browser daemon that spawned ``chromium_pid``, or ``None`` + for a human-started (dock) instance. Chromium itself gets a scrubbed environment, so the daemon's + ``/proc//environ`` is the marker (Linux-only, same user).""" + try: + with open(f"/proc/{chromium_pid}/status", encoding="utf-8") as fh: + ppid = next((int(line.split()[1]) for line in fh if line.startswith("PPid:")), 0) + with open(f"/proc/{ppid}/environ", "rb") as fh: + raw = fh.read() + except (OSError, ValueError): + return None + for item in raw.split(b"\0"): + key, sep, value = item.partition(b"=") + if sep and key == b"AGENT_BROWSER_SESSION": + return value.decode("utf-8", "replace") or None + return None + + +def _pid_alive(pid: int) -> bool: + import psutil + return psutil.pid_exists(pid) + + def env_for_agent(env: dict) -> dict: """Pin agent-browser to the screen's browser identity unless the user pinned their own.""" env.setdefault("AGENT_BROWSER_PROFILE", str(profile_dir())) diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py index 16e4569bfb23..a9ce1923f099 100644 --- a/tools/bot_desktop/runtime.py +++ b/tools/bot_desktop/runtime.py @@ -327,10 +327,9 @@ def _spawn_and_wait(sd: Path, num: int, wait_seconds: float) -> DesktopStatus: "HERMES_BD_CONFIG_HOME": str(sd / "xdg"), "HERMES_BD_GEOMETRY": geometry(), }) - from tools.bot_desktop.browser import dock_launch + from tools.bot_desktop.browser import dock_command, dock_launch if (browser := dock_launch()) is not None: - # first-run / default-browser dialogs would sit between the human and the bot's tabs - child_env["HERMES_BD_BROWSER_EXEC"] = f"{browser[0]} --user-data-dir={browser[1]} --no-first-run --no-default-browser-check" + child_env["HERMES_BD_BROWSER_EXEC"] = dock_command(*browser) # Truncated per start: the log is a diagnostic for THIS launch, and nothing rotates it otherwise. log = open(sd / "launcher.log", "wb") # noqa: SIM115 — handed to the child, closed by it proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host) diff --git a/tools/browser_tool_session.py b/tools/browser_tool_session.py index 574491592349..e762d191075e 100644 --- a/tools/browser_tool_session.py +++ b/tools/browser_tool_session.py @@ -604,6 +604,15 @@ def _shares_bot_desktop_browser(session_info: Dict[str, Any]) -> bool: return bool(_bd_runtime.published_env().get("DISPLAY")) or _bd_lease.human_holds() +def _bot_desktop_attach_port(session_info: Dict[str, Any]) -> Optional[int]: + """DevTools port of a human-started Chromium on the Bot Desktop's shared profile, else ``None``.""" + if not _shares_bot_desktop_browser(session_info): + return None + from tools.bot_desktop import browser as _bd_browser + return _bd_browser.running_instance_cdp_port(str(_bd_browser.profile_dir()), + exclude_session=session_info["session_name"]) + + def _run_browser_command_unfenced(task_id: str, command: str, args: List[str], timeout: int, _engine_override: Optional[str], browser_cmd, session_info: Dict[str, Any]) -> Dict[str, Any]: # Cleanup stops the supervisor before closing the backend; keep it stopped. @@ -619,6 +628,12 @@ def _run_browser_command_unfenced(task_id: str, command: str, args: List[str], t backend_args = ["--cdp", session_info["cdp_url"]] else: backend_args = ["--session", session_info["session_name"]] + if (bd_port := _bot_desktop_attach_port(session_info)) is not None: + # A Chromium already runs on the Bot Desktop's shared profile (the human clicked the dock's + # Browser first): a launch would be forwarded into it by Chromium's singleton and die without + # a DevTools endpoint, so the session's daemon attaches to the port it advertises instead. + # Same daemon (keyed by --session) either way, so snapshot refs stay valid across commands. + backend_args += ["--cdp", str(bd_port)] if _cloud._is_headed_mode(): backend_args.append("--headed") if engine != "auto" and not _bt._is_camofox_mode(): From 8fa8c5e83353afdef2542a3e7e24a020d74d1750 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:31:27 -0700 Subject: [PATCH 39/55] fix(bot-screen): Xvnc stops broadcasting the screen clipboard to watchers TigerVNC sends every clipboard change on the desktop to ALL connected RFB clients by default, so whatever the person in control copied (a password manager entry, a 2FA code) landed in every watcher's clipboard too. -SendCutText=0 turns that off; -AcceptCutText stays at its default so pasting INTO the screen keeps working. The comment above the Xvnc line claimed the 0600 socket is reachable "solely by the gateway process"; it is reachable by any process running as this user, which is the actual boundary. Live: runtime.start() on this host (Xtigervnc 1.15) comes up with the flag, xdpyinfo answers, the launcher log shows no option error. (cherry picked from commit 45cd0ce36a55ea648bac479a43547637b547911b) --- tests/tools/test_bot_desktop_launcher_xvnc.py | 41 +++++++++++++++++++ tools/bot_desktop/launcher.sh | 7 +++- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 tests/tools/test_bot_desktop_launcher_xvnc.py diff --git a/tests/tools/test_bot_desktop_launcher_xvnc.py b/tests/tools/test_bot_desktop_launcher_xvnc.py new file mode 100644 index 000000000000..c2861796c249 --- /dev/null +++ b/tests/tools/test_bot_desktop_launcher_xvnc.py @@ -0,0 +1,41 @@ +"""The Bot Desktop's Xvnc is started with the RFB options the lease design relies on.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +LAUNCHER = Path(__file__).resolve().parents[2] / "tools" / "bot_desktop" / "launcher.sh" +pytestmark = pytest.mark.linux_only + + +def test_xvnc_never_sends_the_holders_clipboard_to_watchers(tmp_path): + """Whoever holds control may paste INTO the screen (AcceptCutText), but the screen's clipboard must + not be pushed to every connected viewer (SendCutText off): watchers are not the holder.""" + bindir = tmp_path / "bin" + bindir.mkdir() + argv_log = tmp_path / "xvnc-argv" + (bindir / "Xvnc").write_text(f'#!/bin/sh\nprintf "%s\\n" "$@" > "{argv_log}"\nexec sleep 3\n', encoding="utf-8") + for stub in ("xdpyinfo", "setxkbmap", "xsetroot", "xset", "dbus-run-session"): + (bindir / stub).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + for exe in bindir.iterdir(): + exe.chmod(0o755) + for tool in ("mkdir", "sed", "cat", "printf", "dirname", "bash", "sh", "rm", "ln", "touch", "chmod", "xauth", "od", "tr", "awk", "seq", "sleep", "kill"): + real = shutil.which(tool) + if real and not (bindir / tool).exists(): + (bindir / tool).symlink_to(real) + env = { + "PATH": str(bindir), "HOME": str(tmp_path), + "HERMES_BD_PROFILE": "t", "HERMES_BD_DISPLAY_NUM": "99", + "HERMES_BD_SOCKET": str(tmp_path / "rfb.sock"), "HERMES_BD_XAUTH": str(tmp_path / "Xauthority"), + "HERMES_BD_ENV_FILE": str(tmp_path / "env"), "HERMES_BD_CONFIG_HOME": str(tmp_path / "xdg"), + } + subprocess.run(["bash", str(LAUNCHER)], env=env, check=True, stdin=subprocess.DEVNULL, capture_output=True, timeout=30) + argv = argv_log.read_text(encoding="utf-8").split("\n") + assert "-SendCutText=0" in argv, argv + assert not any(a.startswith("-AcceptCutText") for a in argv), "paste into the screen must keep working" + assert not os.path.exists(tmp_path / "rfb.sock") # stub never bound it; nothing leaked diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh index 842a2582f53b..9eb106b1ed71 100755 --- a/tools/bot_desktop/launcher.sh +++ b/tools/bot_desktop/launcher.sh @@ -222,10 +222,13 @@ done # ---- X server + RFB (TigerVNC Xvnc), Unix socket only ---- # SecurityTypes None is safe ONLY because -rfbport -1 disables TCP and the 0600 socket is reachable -# solely by the gateway process, whose WebSocket bridge performs the real authentication. +# only by processes running as this user (the gateway's WebSocket bridge does the real authentication; +# same-UID processes, the bot's own terminal tool included, are inside that boundary by design). +# -SendCutText=0: watchers must never receive the holder's clipboard; -AcceptCutText stays on so +# paste INTO the screen keeps working. Xvnc "$DISPLAY" -geometry "$GEOM" -depth "$DEPTH" -dpi 96 \ -rfbport -1 -rfbunixpath "$HERMES_BD_SOCKET" -rfbunixmode 0600 \ - -SecurityTypes None -AlwaysShared -AcceptSetDesktopSize -FrameRate 30 \ + -SecurityTypes None -AlwaysShared -AcceptSetDesktopSize -FrameRate 30 -SendCutText=0 \ -desktop "hermes:$HERMES_BD_PROFILE" -auth "$XAUTHORITY" -nolisten tcp \ -Log '*:stderr:30' & XVNC_PID=$! From e7477a9e5efedb7244460a07e2040b4628273e6f Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:33:43 -0700 Subject: [PATCH 40/55] docs(bot-screen): threat model, lease-file semantics, pane-close vs dropped link, recovery, Fedora packages The page overclaimed ("the bot never sees what you type") and understated the boundary: the lease is a tool-level fence on computer_use and the browser tools, and every same-UID process (other bots, the bot's own terminal tool) can reach the RFB socket, the X display and the lease file. Say so, and say per-bot OS users are out of scope. - Lease file: absent = bot holds (fresh profile); present but unreadable/corrupt = fails closed (matches tools/bot_desktop/lease.py::_read). - Pane close hands back; a DROPPED connection keeps the human's exclusion until they reconnect or hand back; "Hand back (force)" after a reload. - Clipboard: Xvnc runs with -SendCutText=0, watchers never receive the holder's clipboard. - Recovery text names `hermes computer-use screen stop` as releasing the lease and stopping. - Fedora row lists the per-binary packages actually needed (xsetroot xset xdpyinfo xprop setxkbmap ...) instead of the retired xorg-x11-server-utils / xorg-x11-utils metapackages; xprop added to the Arch row since the launcher waits on it. (cherry picked from commit 1c93b0645353eb485fd17aa1076d163ce3e897d9) --- .../docs/user-guide/features/bot-screen.md | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md index 363bc369111d..f201fbf775ee 100644 --- a/website/docs/user-guide/features/bot-screen.md +++ b/website/docs/user-guide/features/bot-screen.md @@ -18,6 +18,14 @@ its own cookies. Screens are work surfaces, not security boundaries: the bots share the host's user account, files and network (the same model as other hosted-agent products). +**Threat model.** The screen's RFB socket, the X display, the browser profile +and the control-lease file all belong to the gateway's OS user. Any process +running as that user — another bot on the same host, and the bot's own +`terminal` tool included — can reach them directly, bypassing the pane and the +lease. The lease is a tool-level fence on `computer_use` and the browser tools, +not an OS one. Running each bot as its own OS user is out of scope; if that +isolation matters to you, put the bots on separate hosts. + ## Requirements - The gateway host runs Linux. macOS and Windows hosts already have a real @@ -34,8 +42,8 @@ hosted-agent products). | Distro | Packages | |---|---| | Debian / Ubuntu | `tigervnc-standalone-server xfce4-panel xfwm4 xfdesktop4 xfce4-settings xfce4-terminal dbus-x11 x11-xserver-utils x11-utils xauth fonts-dejavu-core` | - | Fedora | `tigervnc-server-minimal xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal dbus-x11 xorg-x11-server-utils xorg-x11-utils xorg-x11-xauth dejavu-sans-fonts` | - | Arch | `tigervnc xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal xorg-xsetroot xorg-xset xorg-xdpyinfo xorg-xauth xorg-setxkbmap ttf-dejavu` | + | Fedora | `tigervnc-server-minimal xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal dbus-x11 xsetroot xset xdpyinfo xprop xorg-x11-xauth setxkbmap dejavu-sans-fonts` | + | Arch | `tigervnc xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal xorg-xsetroot xorg-xset xorg-xdpyinfo xorg-xprop xorg-xauth xorg-setxkbmap ttf-dejavu` | Deliberately **not** the `xfce4` metapackage: it pulls in the screensaver, power manager and polkit agent that lock or prompt a headless desktop. @@ -65,10 +73,17 @@ Every bot's computer is one click away in three places of Hermes Desktop: 3. Click **Take over**. The border turns red, your keyboard and mouse now drive the bot's screen. Sign in, solve the CAPTCHA, approve the payment. 4. Click **Hand back**. The bot regains control and re-captures the screen - before continuing. Closing the pane also hands control back. - -While you hold control the bot's `computer_use` calls (captures included) are -refused with `human_has_control`; the bot never sees what you type. + before continuing. Closing the pane also hands control back. A *dropped* + connection is different: if your laptop lid closes or Wi-Fi drops while you + hold control, you keep it — the bot stays locked out of a screen you may be + mid-login on — until you reconnect and hand back. If you come back after a + reload and the pane still says a human holds control, a **Hand back (force)** + button appears to clear it. + +While you hold control, the bot's `computer_use` and browser tools are refused +with `human_has_control`, captures included. This is a tool-level fence, not an +OS one: the bot runs as the same user as its screen. Don't type secrets into a +bot you wouldn't trust with them. The bot can ask for you: when it recognises a login or verification step it calls `computer_use` with `action: "request_handoff"` and a reason, the pane @@ -114,7 +129,9 @@ Xauthority, launcher log, per-profile xfconf). - **TigerVNC `Xvnc`** is the X server and the RFB server in one process, per profile, listening only on a `0600` Unix socket. No TCP port, no VNC - password: the gateway is the only process that can reach it. + password: only processes running as the gateway's user can reach it (see the + threat model above), and the gateway's WebSocket bridge is the authenticated + way in. - **Xfce** starts component-wise (`xfsettingsd`, `xfwm4 --compositor=off`, `xfdesktop`, `xfce4-panel`) under a private D-Bus session, without `xfce4-session`, so nothing tries to lock the screen or reach `logind`. @@ -125,7 +142,13 @@ Xauthority, launcher log, per-profile xfconf). and Hermes Cloud connections alike. - **Control lease.** The gateway drops keyboard, pointer and clipboard messages from any viewer that does not hold the lease, at the RFB byte level; noVNC's - view-only flag is only the UI hint. The same lease gates `computer_use`. + view-only flag is only the UI hint. The same lease gates `computer_use` and + the browser tools. It is a file under `/bot-desktop/`: no file + means the bot holds control (a fresh profile); a file that exists but cannot + be read or parsed fails closed — the bot is treated as locked out until the + next successful hand-off rewrites it. Xvnc never pushes the screen's clipboard + to viewers (`-SendCutText=0`), so watchers do not receive what the person in + control copies; pasting into the screen still works. - **Display binding.** The launcher publishes `DISPLAY`, `XAUTHORITY` and the D-Bus address; every cua-driver and headed-browser spawn for that profile inherits them, so the bot never acts on a display a human is sitting at. @@ -140,4 +163,6 @@ Xauthority, launcher log, per-profile xfconf). keysyms and cua-driver agree; change it with `setxkbmap` on that `DISPLAY` if you need another layout. - **Bot says `human_has_control` after you left** — click **Hand back** in the - pane, or `hermes computer-use screen stop` / `start`. + pane (or **Hand back (force)** after a reload). From a shell, + `hermes computer-use screen stop` releases the lease and stops the screen; + `hermes computer-use screen start` brings it back with the bot in control. From a7abbfd70d8ff7a9d13949ff01d6fddb4274c054 Mon Sep 17 00:00:00 2001 From: Yags <166958865+whyyagswhy@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:39:23 -0400 Subject: [PATCH 41/55] fix(bot-screen): hand back on intentional pane closure Send close code 1000 before noVNC can send its statusless close. Keep reconnect teardown statusless so it does not release the human lease. Cover both lifecycle paths with component tests; verify the ordering against real noVNC and Xvnc separately. Addresses the close-code finding discussed by MrD1az, Xipong, and other reviewers on #108914. (cherry picked from commit 6112341f0c79c710387de66db8c309a39e02eaf0) --- .../screen-pane-lifecycle.test.tsx | 129 ++++++++++++++++++ .../src/plugins/hermes-bots/screen-pane.tsx | 11 +- 2 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx new file mode 100644 index 000000000000..55ce8b6e40c1 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx @@ -0,0 +1,129 @@ +import { act, fireEvent, render, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import type { DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +const sockets = vi.hoisted(() => [] as Array<{ closeCodes: number[]; closed: boolean; close: (code?: number) => void }>) + +vi.mock('@hermes/plugin-sdk', async () => { + const { useStore } = await import('@nanostores/react') + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Button: ({ children, ...props }: React.ButtonHTMLAttributes) => ( + + ), + Codicon: () => null, + GlyphSpinner: () => null, + EmptyState: () => null, + useValue: useStore, + host: { onEvent: onGatewayEvent } + } +}) +vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name })) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + title: 'Screen', + youControl: 'You control', + handBack: 'Hand back', + takeOver: 'Take over', + reconnect: 'Reconnect', + streamLost: 'Stream lost' + } + }) +})) +vi.mock('./screen-connection', () => ({ + VIEWER_ID: 'this-viewer', + displayRequest: vi.fn(), + resolveScreenWsUrl: vi.fn(async () => 'ws://localhost/api/display/ws'), + isEventForBotScreen: () => false +})) +vi.mock('@novnc/novnc', () => ({ + default: class { + constructor( + _target: HTMLElement, + private socket: { close: () => void } + ) {} + addEventListener(type: string, callback: () => void) { + if (type === 'connect') { + queueMicrotask(callback) + } + } + // noVNC 1.7 disconnects its WebSocket without a close code. + disconnect() { + this.socket.close() + } + focus() {} + } +})) + +import { displayRequest } from './screen-connection' +import { BotScreenPane } from './screen-pane' +import { $screenState } from './screen-state' + +const bot: RosterRow = { name: 'default' } + +const status: DisplayStatus = { + profile: 'default', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 42, + display: ':20', + socket: '/tmp/rfb.sock', + geometry: '1440x900', + install_command: null, + lease: { holder: 'human', viewer_id: 'this-viewer', pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + $screenState.set({}) + sockets.length = 0 + vi.mocked(displayRequest) + .mockReset() + .mockResolvedValue({ ...status, ticket: 'test-ticket', viewer_id: 'this-viewer' }) + vi.stubGlobal( + 'WebSocket', + class { + closeCodes: number[] = [] + closed = false + constructor() { + sockets.push(this) + } + close(code?: number) { + // Subsequent close calls cannot replace the frame already sent to the server. + if (this.closed) { + return + } + + this.closed = true + this.closeCodes.push(code ?? 1005) + } + } + ) +}) + +afterEach(() => vi.unstubAllGlobals()) + +it('sends an intentional close before noVNC can send its statusless close on pane unmount', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + view.unmount() + expect(sockets[0].closeCodes).toEqual([1000]) +}) + +it('does not hand back while replacing a stream to reconnect the same viewer', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + fireEvent.click(view.getByTitle('Reconnect')) + await waitFor(() => expect(sockets).toHaveLength(2)) + expect(sockets[0].closeCodes).toEqual([1005]) + expect(sockets[1].closed).toBe(false) + view.unmount() +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index bfe65b6d69a4..e81fc263331e 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -78,8 +78,15 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { }) }, [bot, refresh, status?.profile_key]) - const detach = useCallback(() => { + const detach = useCallback((handBack = false) => { attachGeneration.current += 1 + + // noVNC closes without a status. Intentional pane closure must send 1000 + // first; reconnect teardown must keep the human lease instead. + if (handBack) { + socket.current?.close(1000) + } + rfb.current?.disconnect() rfb.current = null socket.current?.close() @@ -158,7 +165,7 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { // Visibility is not lifecycle: the stream stays attached while the pane is // hidden; only unmount tears it down (and hands control back server-side). - useEffect(() => () => detach(), [detach]) + useEffect(() => () => detach(true), [detach]) useEffect(() => { if (status?.running && conn === 'idle') { From 07ee4e92116eda4d89de4ea8c24720fa3b29c52d Mon Sep 17 00:00:00 2001 From: Yags <166958865+whyyagswhy@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:39:23 -0400 Subject: [PATCH 42/55] fix(bot-screen): keep portal state and previews with their owner Use the connection/profile event predicate in the portal, never match a legacy group to a remote namesake, and reset thumbnail state on owner changes. Component tests cover cross-host events, click routing, pending/rejected thumbnails, and stale responses. Addresses findings from Julientalbot, erosika, and BearHuddleston on #108914. (cherry picked from commit 65155c92a808ce5bb94d783bb0f42d6a4fc7d94f) --- .../src/plugins/hermes-bots/screen-hero.tsx | 6 + .../hermes-bots/screen-isolation.test.tsx | 167 ++++++++++++++++++ .../src/plugins/hermes-bots/screen-portal.tsx | 6 +- 3 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx diff --git a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx index 32bca2d1fd6e..2e108c20c24a 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx @@ -10,6 +10,7 @@ import { Codicon } from '@hermes/plugin-sdk' import { useEffect, useRef, useState } from 'react' +import { botSelectionKey } from './data' import { useBots } from './i18n' import { displayRequest } from './screen-connection' import { openBotScreen } from './screen-open' @@ -100,6 +101,11 @@ const TONE_RING: Partial> = { } export function ScreenHero({ bot, meta }: { bot: RosterRow; meta?: BotMeta | null }) { + // A profile switch must discard the previous owner's pixels before painting. + return +} + +function ScreenHeroContent({ bot, meta }: { bot: RosterRow; meta?: BotMeta | null }) { const t = useBots() const { tone } = useScreenPortalState(bot) const running = tone === 'live' || tone === 'human' || tone === 'other' diff --git a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx new file mode 100644 index 000000000000..609a777ac487 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx @@ -0,0 +1,167 @@ +import { act, fireEvent, render, renderHook } from '@testing-library/react' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import type { RosterRow } from './types' + +vi.mock('@hermes/plugin-sdk', async () => { + const { useStore } = await import('@nanostores/react') + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Codicon: () => null, + useValue: useStore, + resolveSiblingWsUrl: vi.fn(), + host: { onEvent: onGatewayEvent, requestProfile: vi.fn() } + } +}) +vi.mock('./data', async () => { + const { atom } = await import('nanostores') + + return { + $lastRoster: atom([]), + botSelectionKey: (bot: RosterRow) => + bot.sourceScoped || bot.remoteSource ? `${bot.connectionId}::${bot.name}` : bot.name + } +}) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + portalTitle: 'Screen', + portalWatching: 'Live', + portalYouControl: 'You control', + portalOtherControls: 'Other viewer', + heroOpenLive: 'Open live', + heroStale: 'Last seen', + heroConnecting: 'Connecting' + } + }) +})) +vi.mock('./screen-open', () => ({ openBotScreen: vi.fn() })) + +import { host } from '@hermes/plugin-sdk' + +// Exercise the real event bus in this integration test, not a copied dispatcher. +// eslint-disable-next-line no-restricted-imports +import { emitGatewayEvent } from '../../contrib/events' + +import { $lastRoster } from './data' +import { type DisplayStatus, VIEWER_ID } from './screen-connection' +import { ScreenHero } from './screen-hero' +import { openBotScreen } from './screen-open' +import { ProfileGroupScreenPortal, useScreenPortalState } from './screen-portal' +import { $screenState, setScreenStatus } from './screen-state' + +const botA: RosterRow = { name: 'default', sourceScoped: true, connectionId: 'host-a', connectionKind: 'remote' } +const botB: RosterRow = { ...botA, connectionId: 'host-b' } + +const status: DisplayStatus = { + profile: 'default', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 42, + display: ':20', + socket: '/tmp/rfb.sock', + geometry: '1440x900', + install_command: null, + lease: { holder: 'agent', viewer_id: null, pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + $screenState.set({}) + $lastRoster.set([]) + vi.mocked(host.requestProfile).mockReset() + vi.mocked(openBotScreen).mockClear() + vi.spyOn(globalThis.document, 'hidden', 'get').mockReturnValue(false) +}) + +afterEach(() => { + vi.restoreAllMocks() + vi.useRealTimers() +}) + +it('applies lease events only from the owning host even when profile paths match', () => { + setScreenStatus(botA, status) + const view = renderHook(() => useScreenPortalState(botA)) + const human = { ...status.lease, holder: 'human' as const, viewer_id: VIEWER_ID } + + const emit = (connectionId: string, profileKey = status.profile_key) => + act(() => + emitGatewayEvent({ + type: 'display.lease', + connectionId, + profile: 'default', + payload: { profile_key: profileKey, lease: human } + }) + ) + + emit('host-b') + expect(view.result.current.lease?.holder).toBe('agent') + emit('host-a', '/another/profile') + expect(view.result.current.lease?.holder).toBe('agent') + emit('host-a') + expect(view.result.current.lease).toEqual(human) + view.unmount() +}) + +it('does not match a legacy profile group to a same-named remote bot', () => { + const legacy: RosterRow = { name: 'default' } + $lastRoster.set([botB, legacy]) + setScreenStatus(botB, status) + setScreenStatus(legacy, status) + const view = render() + + fireEvent.click(view.getByRole('button')) + expect(openBotScreen).toHaveBeenCalledWith(legacy, null) + view.rerender() + fireEvent.click(view.getByRole('button')) + expect(openBotScreen).toHaveBeenLastCalledWith(botB, null) + view.unmount() +}) + +it.each(['pending', 'rejected'])('never displays host A pixels under host B while B is %s', async state => { + setScreenStatus(botA, status) + setScreenStatus(botB, status) + vi.mocked(host.requestProfile) + .mockResolvedValueOnce({ data_url: 'data:image/jpeg;base64,HOST_A' }) + .mockImplementationOnce(() => (state === 'pending' ? new Promise(() => {}) : Promise.reject(new Error('offline')))) + const view = render() + await act(async () => {}) + expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_A') + + view.rerender() + await act(async () => {}) + expect(view.container.querySelector('img')).toBeNull() + view.unmount() +}) + +it('discards a late thumbnail from the previous owner and retains a same-owner frame on refresh failure', async () => { + vi.useFakeTimers() + setScreenStatus(botA, status) + setScreenStatus(botB, status) + let finishA!: (value: unknown) => void + vi.mocked(host.requestProfile) + .mockImplementationOnce( + () => + new Promise(resolve => { + finishA = resolve + }) + ) + .mockResolvedValueOnce({ data_url: 'data:image/jpeg;base64,HOST_B' }) + .mockRejectedValue(new Error('offline')) + const view = render() + view.rerender() + await act(async () => {}) + await act(async () => { + finishA({ data_url: 'data:image/jpeg;base64,HOST_A' }) + }) + expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_B') + await act(async () => { + await vi.advanceTimersByTimeAsync(12_000) + }) + expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_B') + expect(view.getByRole('button').getAttribute('aria-label')).toContain('Last seen') + view.unmount() +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx index 3204de79064a..d80a3716f71f 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx @@ -17,7 +17,7 @@ import { useEffect } from 'react' import { $lastRoster } from './data' import { useBots } from './i18n' import { resolveBotConnectionRoute } from './routing' -import { type DisplayLease, displayRequest, type DisplayStatus, VIEWER_ID } from './screen-connection' +import { type DisplayLease, displayRequest, type DisplayStatus, isEventForBotScreen, VIEWER_ID } from './screen-connection' import { openBotScreen } from './screen-open' import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' import type { BotMeta, RosterRow } from './types' @@ -102,7 +102,7 @@ export function useScreenPortalState(bot: RosterRow) { host.onEvent('display.lease', (event: RpcEvent) => { const payload = event.payload as { profile_key?: string; lease?: DisplayLease } | undefined - if (payload?.lease && payload.profile_key && payload.profile_key === profileKey) { + if (payload?.lease && isEventForBotScreen(bot, event, profileKey)) { setScreenLease(bot, payload.lease) } }), @@ -163,7 +163,7 @@ export function ProfileGroupScreenPortal({ route }: { route: ProfileGroupRoute } const resolved = resolveBotConnectionRoute(row) return resolved.route - ? resolved.route.profile === route.profile && (route.connectionId === null || resolved.route.connectionId === route.connectionId) + ? resolved.route.profile === route.profile && resolved.route.connectionId === (route.connectionId ?? 'local') : row.name === route.profile && route.connectionId === null }) ?? (route.connectionId From f1465777ff1349574b3ff6d1284eb5343785d87c Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:24:27 -0700 Subject: [PATCH 43/55] fix(bot-screen): viewer identity is server-minted, "I hold" compares the lease hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `display.observe` now returns the viewer id this attach is known by, and lease payloads name the holder by `viewer_hash` (sha256(viewer_id)[:12]) instead of the raw id. The pane stores the minted id per attach ($screenState.viewer), passes it to display.lease.acquire/release, and derives iHold (pane) and the 'human' portal tone from `leaseHeldBy` — hash compare with a raw-id fallback for backends that still broadcast viewer_id. Why: the client-generated VIEWER_ID constant let a reloaded window mint a new identity while the lease still named the old one, and a raw id on the wire was a usable credential for anyone listening on the profile's event stream. (cherry picked from commit ad985457d185a99f23807c282a9a2eae7b496ecc) --- .../plugins/hermes-bots/screen-connection.ts | 33 ++++- .../hermes-bots/screen-isolation.test.tsx | 4 +- .../screen-pane-lifecycle.test.tsx | 6 +- .../src/plugins/hermes-bots/screen-pane.tsx | 23 +-- .../src/plugins/hermes-bots/screen-portal.tsx | 10 +- .../src/plugins/hermes-bots/screen-state.ts | 30 +++- .../screen-viewer-identity.test.tsx | 135 ++++++++++++++++++ 7 files changed, 216 insertions(+), 25 deletions(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts index 75daaef936b1..e19917bbb46c 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -18,7 +18,10 @@ import type { RosterRow } from './types' export interface DisplayLease { holder: 'agent' | 'human' + /** Raw holder id — only older backends still broadcast it; newer ones send `viewer_hash`. */ viewer_id: null | string + /** First 12 hex of sha256(viewer_id): names the holder without leaking a usable id. */ + viewer_hash?: null | string since: number reason: string pending_handoff: null | string @@ -42,12 +45,36 @@ export interface DisplayStatus { export interface DisplayObserveResult extends DisplayStatus { ticket: string path: string + /** Server-minted per attach: the only id the lease will ever be compared against. */ viewer_id: string } -/** Stable per-window viewer identity: the lease names who holds control, and a - * reload must NOT silently inherit a stale holder's authority. */ -export const VIEWER_ID = `desktop-${Math.random().toString(36).slice(2, 10)}` +/** This window's identity for one attach: the minted id plus its lease-payload hash. */ +export interface ScreenViewer { + id: string + hash: string +} + +const VIEWER_HASH_HEX = 12 + +/** `viewer_hash` as the lease broadcasts it: first 12 hex of sha256(viewer_id). */ +export async function viewerHash(viewerId: string): Promise { + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(viewerId)) + + return Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, '0')) + .join('') + .slice(0, VIEWER_HASH_HEX) +} + +/** Does `viewer` (this window's attach) hold `lease`? Newer backends name the + * holder by hash only; a payload still carrying the raw id is compared raw. */ +export function leaseHeldBy(lease: DisplayLease | null | undefined, viewer: ScreenViewer | null | undefined): boolean { + if (!lease || !viewer || lease.holder !== 'human') { + return false + } + + return lease.viewer_id != null ? lease.viewer_id === viewer.id : lease.viewer_hash === viewer.hash +} /** Bare-profile fallback so a v1 local bot (no registry route) still resolves. */ export function botScreenRoute(bot: RosterRow): PluginProfileRoute | string { diff --git a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx index 609a777ac487..ce342893ad19 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx @@ -45,7 +45,7 @@ import { host } from '@hermes/plugin-sdk' import { emitGatewayEvent } from '../../contrib/events' import { $lastRoster } from './data' -import { type DisplayStatus, VIEWER_ID } from './screen-connection' +import type { DisplayStatus } from './screen-connection' import { ScreenHero } from './screen-hero' import { openBotScreen } from './screen-open' import { ProfileGroupScreenPortal, useScreenPortalState } from './screen-portal' @@ -85,7 +85,7 @@ afterEach(() => { it('applies lease events only from the owning host even when profile paths match', () => { setScreenStatus(botA, status) const view = renderHook(() => useScreenPortalState(botA)) - const human = { ...status.lease, holder: 'human' as const, viewer_id: VIEWER_ID } + const human = { ...status.lease, holder: 'human' as const, viewer_id: 'this-viewer' } const emit = (connectionId: string, profileKey = status.profile_key) => act(() => diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx index 55ce8b6e40c1..bfab9a0d38d7 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx @@ -2,6 +2,7 @@ import { act, fireEvent, render, waitFor } from '@testing-library/react' import { afterEach, beforeEach, expect, it, vi } from 'vitest' import type { DisplayStatus } from './screen-connection' +import type * as ScreenConnection from './screen-connection' import type { RosterRow } from './types' const sockets = vi.hoisted(() => [] as Array<{ closeCodes: number[]; closed: boolean; close: (code?: number) => void }>) @@ -34,8 +35,9 @@ vi.mock('./i18n', () => ({ } }) })) -vi.mock('./screen-connection', () => ({ - VIEWER_ID: 'this-viewer', +vi.mock('./screen-connection', async importActual => ({ + // Real pure helpers (viewerHash / leaseHeldBy); only the gateway legs are faked. + ...(await importActual()), displayRequest: vi.fn(), resolveScreenWsUrl: vi.fn(async () => 'ws://localhost/api/display/ws'), isEventForBotScreen: () => false diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index e81fc263331e..bfd9ce0e44f9 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -15,9 +15,9 @@ import type { RpcEvent } from '@hermes/plugin-sdk' import { useCallback, useEffect, useRef, useState } from 'react' import { useBots } from './i18n' -import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isEventForBotScreen, resolveScreenWsUrl, VIEWER_ID } from './screen-connection' +import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isEventForBotScreen, leaseHeldBy, resolveScreenWsUrl, viewerHash } from './screen-connection' import { ScreenInstallCard } from './screen-install' -import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenViewer } from './screen-state' import type { RosterRow } from './types' type RfbLike = { @@ -46,7 +46,10 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { const state = screenStateFor(screen, bot) const status = state?.status ?? null const lease = state?.lease ?? status?.lease ?? null - const iHold = lease?.holder === 'human' && lease.viewer_id === VIEWER_ID + // The server mints this window's viewer id per attach (`display.observe`); the lease + // names its holder by hash, so a reload can never inherit a stale holder's authority. + const viewer = state?.viewer ?? null + const iHold = leaseHeldBy(lease, viewer) const canvasHost = useRef(null) const rfb = useRef(null) @@ -107,7 +110,8 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { // Load the client BEFORE dialing: noVNC's Websock installs its own `onopen`, so a socket that // opened while the dynamic import was still in flight never hands it the open event. const Rfb = await loadRfb() - const observe = await displayRequest(bot, 'display.observe', { viewer_id: VIEWER_ID }) + const observe = await displayRequest(bot, 'display.observe') + const minted = { id: observe.viewer_id, hash: await viewerHash(observe.viewer_id) } setScreenStatus(bot, observe) const url = await resolveScreenWsUrl(bot, observe.ticket) @@ -115,6 +119,7 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { return } + setScreenViewer(bot, minted) const ws = new WebSocket(url) ws.binaryType = 'arraybuffer' socket.current = ws @@ -124,7 +129,7 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { client.focusOnClick = true client.background = 'transparent' client.qualityLevel = 7 - client.viewOnly = !(observe.lease.holder === 'human' && observe.lease.viewer_id === VIEWER_ID) + client.viewOnly = !leaseHeldBy(observe.lease, minted) client.addEventListener('connect', () => { if (generation === attachGeneration.current) { setConn('live') @@ -201,7 +206,7 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { setBusy(true) try { - const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.acquire', { viewer_id: VIEWER_ID }) + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.acquire', { viewer_id: viewer?.id }) setScreenLease(bot, result.lease) if (conn !== 'live') { @@ -212,20 +217,20 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { } finally { setBusy(false) } - }, [attach, bot, conn]) + }, [attach, bot, conn, viewer?.id]) const handBack = useCallback(async () => { setBusy(true) try { - const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', { viewer_id: VIEWER_ID }) + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', { viewer_id: viewer?.id }) setScreenLease(bot, result.lease) } catch (err) { setError(err instanceof Error ? err.message : String(err)) } finally { setBusy(false) } - }, [bot]) + }, [bot, viewer?.id]) if (status && !status.supported) { return diff --git a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx index d80a3716f71f..2587f284e530 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx @@ -17,15 +17,15 @@ import { useEffect } from 'react' import { $lastRoster } from './data' import { useBots } from './i18n' import { resolveBotConnectionRoute } from './routing' -import { type DisplayLease, displayRequest, type DisplayStatus, isEventForBotScreen, VIEWER_ID } from './screen-connection' +import { type DisplayLease, displayRequest, type DisplayStatus, isEventForBotScreen, leaseHeldBy, type ScreenViewer } from './screen-connection' import { openBotScreen } from './screen-open' import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' import type { BotMeta, RosterRow } from './types' export type PortalTone = 'live' | 'human' | 'other' | 'off' | 'missing' | 'unsupported' | 'unknown' -/** Pure: map cached status + lease to what the portal says. */ -export function portalTone(status: DisplayStatus | null, lease: DisplayLease | null): PortalTone { +/** Pure: map cached status + lease (+ this window's minted viewer, if attached) to what the portal says. */ +export function portalTone(status: DisplayStatus | null, lease: DisplayLease | null, viewer: ScreenViewer | null = null): PortalTone { if (!status) { return 'unknown' } @@ -43,7 +43,7 @@ export function portalTone(status: DisplayStatus | null, lease: DisplayLease | n } if (lease?.holder === 'human') { - return lease.viewer_id === VIEWER_ID ? 'human' : 'other' + return leaseHeldBy(lease, viewer) ? 'human' : 'other' } return 'live' @@ -109,7 +109,7 @@ export function useScreenPortalState(bot: RosterRow) { [bot, profileKey] ) - return { status, lease: state?.lease ?? null, tone: portalTone(status, state?.lease ?? null) } + return { status, lease: state?.lease ?? null, tone: portalTone(status, state?.lease ?? null, state?.viewer ?? null) } } export function ScreenPortal({ bot, meta, compact = false }: { bot: RosterRow; meta?: BotMeta | null; compact?: boolean }) { diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.ts index e93ddea0ef75..71e9ebfcc8af 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-state.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.ts @@ -7,12 +7,14 @@ import { atom } from 'nanostores' import { botSelectionKey } from './data' -import type { DisplayLease, DisplayStatus } from './screen-connection' +import type { DisplayLease, DisplayStatus, ScreenViewer } from './screen-connection' import type { RosterRow } from './types' export interface BotScreenState { status: DisplayStatus | null lease: DisplayLease | null + /** This window's server-minted identity for the bot's current attach; null until the pane observes. */ + viewer: ScreenViewer | null } export const $screenState = atom>({}) @@ -24,7 +26,8 @@ export function screenStateFor(all: Record, bot: RosterR export function setScreenStatus(bot: RosterRow, status: DisplayStatus): void { const key = botSelectionKey(bot) const current = $screenState.get() - $screenState.set({ ...current, [key]: { status, lease: status.lease ?? current[key]?.lease ?? null } }) + const prev = current[key] + $screenState.set({ ...current, [key]: { status, lease: status.lease ?? prev?.lease ?? null, viewer: prev?.viewer ?? null } }) } export function setScreenLease(bot: RosterRow, lease: DisplayLease): void { @@ -32,9 +35,28 @@ export function setScreenLease(bot: RosterRow, lease: DisplayLease): void { const current = $screenState.get() const prev = current[key] - if (prev?.lease && prev.lease.holder === lease.holder && prev.lease.viewer_id === lease.viewer_id && prev.lease.pending_handoff === lease.pending_handoff) { + if ( + prev?.lease && + prev.lease.holder === lease.holder && + prev.lease.viewer_id === lease.viewer_id && + prev.lease.viewer_hash === lease.viewer_hash && + prev.lease.pending_handoff === lease.pending_handoff + ) { + return + } + + $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease, viewer: prev?.viewer ?? null } }) +} + +/** Record the identity `display.observe` minted for this window's attach to `bot`. */ +export function setScreenViewer(bot: RosterRow, viewer: ScreenViewer | null): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + + if ((prev?.viewer ?? null) === viewer) { return } - $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease } }) + $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease: prev?.lease ?? null, viewer } }) } diff --git a/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx new file mode 100644 index 000000000000..2eeb0c32e216 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx @@ -0,0 +1,135 @@ +/** + * Viewer identity is SERVER-MINTED: `display.observe` returns the id this attach + * is known by, and lease payloads name the holder by `viewer_hash` + * (sha256(viewer_id)[:12]) rather than the raw id. "I hold" must be derived + * from the minted id, never from a client-generated constant — otherwise a + * Desktop reload could claim (or lose) control it does not have. + */ + +import { act, render, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import type { DisplayStatus } from './screen-connection' +import type * as ScreenConnection from './screen-connection' +import type { RosterRow } from './types' + +vi.mock('@hermes/plugin-sdk', async () => { + const { useStore } = await import('@nanostores/react') + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Button: ({ children, ...props }: React.ButtonHTMLAttributes) => , + Codicon: () => null, + GlyphSpinner: () => null, + EmptyState: () => null, + useValue: useStore, + host: { onEvent: onGatewayEvent } + } +}) +vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name })) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + title: 'Screen', + youControl: 'You control', + otherControls: 'Other controls', + agentControls: 'Bot controls', + handBack: 'Hand back', + takeOver: 'Take over', + reconnect: 'Reconnect', + streamLost: 'Stream lost' + } + }) +})) +vi.mock('./screen-connection', async importActual => ({ + ...(await importActual()), + displayRequest: vi.fn(), + resolveScreenWsUrl: vi.fn(async () => 'ws://localhost/api/display/ws'), + isEventForBotScreen: () => true +})) +vi.mock('@novnc/novnc', () => ({ + default: class { + addEventListener() {} + disconnect() {} + focus() {} + } +})) + +// Real event bus, so the pane's listener path is the one under test. +// eslint-disable-next-line no-restricted-imports +import { emitGatewayEvent } from '../../contrib/events' + +import { displayRequest, viewerHash } from './screen-connection' +import { BotScreenPane } from './screen-pane' +import { $screenState } from './screen-state' + +const bot: RosterRow = { name: 'default' } +const MINTED = 'srv-viewer-0001' + +const status: DisplayStatus = { + profile: 'default', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 42, + display: ':20', + socket: '/tmp/rfb.sock', + geometry: '1440x900', + install_command: null, + lease: { holder: 'agent', viewer_id: null, viewer_hash: null, pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + $screenState.set({}) + vi.mocked(displayRequest) + .mockReset() + .mockImplementation(async (_bot, method) => + method === 'display.observe' ? { ...status, ticket: 'test-ticket', viewer_id: MINTED } : status + ) + vi.stubGlobal( + 'WebSocket', + class { + binaryType = '' + close() {} + } + ) +}) + +afterEach(() => vi.unstubAllGlobals()) + +const emitLease = (viewer_hash: string) => + act(() => + emitGatewayEvent({ + type: 'display.lease', + payload: { profile_key: status.profile_key, lease: { ...status.lease, holder: 'human', viewer_hash } } + }) + ) + +it('holds control when the lease names the hash of the server-minted viewer id, not when it names another', async () => { + const view = render() + await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe')) + await act(async () => {}) + + emitLease(await viewerHash('someone-else')) + expect(view.queryByText('You control')).toBeNull() + expect(view.getByText('Other controls')).toBeTruthy() + + emitLease(await viewerHash(MINTED)) + expect(view.getByText('You control')).toBeTruthy() + view.unmount() +}) + +it('hands back with the minted id, never a client-generated one', async () => { + const view = render() + await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe')) + await act(async () => {}) + emitLease(await viewerHash(MINTED)) + + await act(async () => { + view.getByText('Hand back').click() + }) + expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.lease.release', { viewer_id: MINTED }) + view.unmount() +}) From cda9e8a1bd014f224aa4b766149e080432072997 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:25:35 -0700 Subject: [PATCH 44/55] fix(bot-screen): hero says why the preview is hidden while a human holds control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `display.thumbnail` answers `{data_url: null, suppressed: 'human_has_control'}` while a human holds the lease. The hero captions that as "Hidden while someone has control" (screen.heroSuppressed, all four locales) and treats it as a successful refresh, so the miss counter never ages a withheld frame into "Last seen — screen unreachable". (cherry picked from commit 7da75f086fe30f0c7e329bba70f32b2dca283334) --- apps/desktop/src/plugins/hermes-bots/i18n.ts | 5 +++++ .../plugins/hermes-bots/screen-connection.ts | 6 ++++++ .../src/plugins/hermes-bots/screen-hero.tsx | 15 ++++++++++----- .../hermes-bots/screen-isolation.test.tsx | 17 +++++++++++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index 637a9ee4b9fb..2d1786b18d91 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -243,6 +243,7 @@ type BotsMessages = { heroNotInstalled: string heroConnecting: string heroStale: string + heroSuppressed: string heroOpenLive: string heroInstall: string heroStart: string @@ -510,6 +511,7 @@ const en: BotsMessages = { heroNotInstalled: 'Not installed on this host', heroConnecting: 'Checking the screen…', heroStale: 'Last seen — screen unreachable', + heroSuppressed: 'Hidden while someone has control', heroOpenLive: 'Open live', heroInstall: 'Install', heroStart: 'Start', @@ -772,6 +774,7 @@ const ja: BotsMessages = { heroNotInstalled: 'このホストには未インストール', heroConnecting: '画面を確認中…', heroStale: '最終表示 — 画面に接続できません', + heroSuppressed: '他の人が操作中は非表示', heroOpenLive: 'ライブで開く', heroInstall: 'インストール', heroStart: '開始', @@ -1029,6 +1032,7 @@ const zh: BotsMessages = { heroNotInstalled: '此主机未安装', heroConnecting: '正在检查屏幕…', heroStale: '最后画面 — 屏幕无法访问', + heroSuppressed: '有人控制时隐藏', heroOpenLive: '实时打开', heroInstall: '安装', heroStart: '启动', @@ -1286,6 +1290,7 @@ const zhHant: BotsMessages = { heroNotInstalled: '此主機未安裝', heroConnecting: '正在檢查螢幕…', heroStale: '最後畫面 — 螢幕無法連線', + heroSuppressed: '有人控制時隱藏', heroOpenLive: '即時開啟', heroInstall: '安裝', heroStart: '啟動', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts index e19917bbb46c..90dfd846260e 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -42,6 +42,12 @@ export interface DisplayStatus { lease: DisplayLease } +export interface DisplayThumbnail { + data_url: string | null + /** Set while a human holds the screen: the frame is withheld, not missing. */ + suppressed?: 'human_has_control' | null +} + export interface DisplayObserveResult extends DisplayStatus { ticket: string path: string diff --git a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx index 2e108c20c24a..93ad3e1bddc0 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx @@ -12,7 +12,7 @@ import { useEffect, useRef, useState } from 'react' import { botSelectionKey } from './data' import { useBots } from './i18n' -import { displayRequest } from './screen-connection' +import { displayRequest, type DisplayThumbnail } from './screen-connection' import { openBotScreen } from './screen-open' import { type PortalTone, useScreenPortalState } from './screen-portal' import type { BotMeta, RosterRow } from './types' @@ -25,12 +25,16 @@ function useLiveThumbnail(bot: RosterRow, running: boolean) { // Consecutive failed refreshes; past STALE_AFTER the frame is shown dimmed as "last seen" so a // dead gateway never keeps looking live. Success resets it. const [misses, setMisses] = useState(0) + // The backend withholds frames while a human holds the screen; that is a + // deliberate answer, not a failed refresh, so it never ages into "stale". + const [suppressed, setSuppressed] = useState(false) const boxRef = useRef(null) useEffect(() => { if (!running) { setDataUrl(null) setMisses(0) + setSuppressed(false) return } @@ -61,10 +65,11 @@ function useLiveThumbnail(bot: RosterRow, running: boolean) { return } - void displayRequest<{ data_url: string | null }>(bot, 'display.thumbnail') + void displayRequest(bot, 'display.thumbnail') .then(result => { if (!cancelled) { setDataUrl(result.data_url) + setSuppressed(result.suppressed === 'human_has_control') setMisses(0) } }) @@ -92,7 +97,7 @@ function useLiveThumbnail(bot: RosterRow, running: boolean) { } }, [bot, running]) - return { dataUrl, boxRef, stale: misses >= STALE_AFTER } + return { dataUrl, boxRef, stale: misses >= STALE_AFTER, suppressed } } const TONE_RING: Partial> = { @@ -109,13 +114,13 @@ function ScreenHeroContent({ bot, meta }: { bot: RosterRow; meta?: BotMeta | nul const t = useBots() const { tone } = useScreenPortalState(bot) const running = tone === 'live' || tone === 'human' || tone === 'other' - const { dataUrl, boxRef, stale } = useLiveThumbnail(bot, running) + const { dataUrl, boxRef, stale, suppressed } = useLiveThumbnail(bot, running) if (tone === 'unsupported') { return null } - const caption = stale ? t.screen.heroStale : { + const caption = suppressed ? t.screen.heroSuppressed : stale ? t.screen.heroStale : { live: t.screen.portalWatching, human: t.screen.portalYouControl, other: t.screen.portalOtherControls, diff --git a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx index ce342893ad19..926a34d8edd3 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx @@ -32,6 +32,7 @@ vi.mock('./i18n', () => ({ portalOtherControls: 'Other viewer', heroOpenLive: 'Open live', heroStale: 'Last seen', + heroSuppressed: 'Hidden while someone has control', heroConnecting: 'Connecting' } }) @@ -165,3 +166,19 @@ it('discards a late thumbnail from the previous owner and retains a same-owner f expect(view.getByRole('button').getAttribute('aria-label')).toContain('Last seen') view.unmount() }) + +it('captions a suppressed thumbnail as hidden-while-controlled and never ages it into stale', async () => { + vi.useFakeTimers() + setScreenStatus(botA, status) + vi.mocked(host.requestProfile).mockResolvedValue({ data_url: null, suppressed: 'human_has_control' }) + const view = render() + await act(async () => {}) + expect(view.getByRole('button').getAttribute('aria-label')).toContain('Hidden while someone has control') + + await act(async () => { + await vi.advanceTimersByTimeAsync(20_000) + }) + expect(view.getByRole('button').getAttribute('aria-label')).toContain('Hidden while someone has control') + expect(view.getByRole('button').getAttribute('aria-label')).not.toContain('Last seen') + view.unmount() +}) From d17f101866c02bf15ddf4ce6902f1c0ebdd2fa0e Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:27:14 -0700 Subject: [PATCH 45/55] fix(bot-screen): keep the bot's pooled socket open across install and attach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK disposes an INACTIVE registry-routed bot's secondary socket as soon as its request count returns to zero, so `display.install.log/done` and the pane's `display.lease` events had no socket to arrive on — the install card sat at "Installing…" forever and the pane never learned about a take-over. `retainBotScreen` feature-detects `host.retainProfile` (like group-turns). The install card acquires it before `display.install` and releases on done/failed/unmount; the pane holds one for the attach lifetime, released by detach (reconnect swaps it, unmount drops it). (cherry picked from commit 1eed9c82e27d24a63abeda835f3349a9db66ebfe) --- .../plugins/hermes-bots/screen-connection.ts | 23 ++++ .../screen-install-retention.test.tsx | 107 ++++++++++++++++++ .../plugins/hermes-bots/screen-install.tsx | 20 +++- .../screen-pane-lifecycle.test.tsx | 28 ++++- .../src/plugins/hermes-bots/screen-pane.tsx | 16 ++- 5 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts index 90dfd846260e..e77ffea9b801 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -109,6 +109,29 @@ export function displayRequest(bot: RosterRow, method: string, params: Record return host.requestProfile(botScreenRoute(bot), method, params) } +/** + * Hold the bot's pooled gateway socket open across a `display.*` sequence. The + * SDK disposes an inactive registry-routed socket once its request count hits + * zero, so without this the `display.install.*` / `display.lease` events that + * follow the request never arrive. Feature-detected: older hosts (and local + * routes, which never close) get a no-op release. + */ +export async function retainBotScreen(bot: RosterRow): Promise<() => void> { + const noop = () => undefined + + if (typeof host.retainProfile !== 'function') { + return noop + } + + try { + const release = await host.retainProfile(botScreenRoute(bot)) + + return typeof release === 'function' ? release : noop + } catch { + return noop + } +} + /** * Resolve the RFB WebSocket URL for `bot`: the bot's gateway `/api/ws` origin * (fresh credential for OAuth remotes) with the path swapped for the display diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx new file mode 100644 index 000000000000..1f057dacce53 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx @@ -0,0 +1,107 @@ +/** + * An INACTIVE registry-routed bot's pooled socket is disposed by the SDK as soon + * as its request count hits zero — so `display.install.log/done` (and the + * pane's `display.lease` events) would never arrive. The install card must hold + * a retention from before `display.install` until the done event lands. + */ + +import { act, fireEvent, render } from '@testing-library/react' +import { beforeEach, expect, it, vi } from 'vitest' + +import type { DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +const calls = vi.hoisted(() => [] as string[]) + +vi.mock('@hermes/plugin-sdk', async () => { + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Button: ({ children, ...props }: React.ButtonHTMLAttributes) => , + Codicon: () => null, + GlyphSpinner: () => null, + resolveSiblingWsUrl: vi.fn(), + host: { + onEvent: onGatewayEvent, + requestProfile: vi.fn(async (_route: unknown, method: string) => { + calls.push(`request:${method}`) + + return {} + }), + retainProfile: vi.fn(async () => { + calls.push('retain') + + return () => { + calls.push('release') + } + }) + } + } +}) +vi.mock('./routing', () => ({ + botConnectionRoute: () => ({ connectionId: 'host-a', profile: 'ops', targetProfile: 'ops' }) +})) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + notInstalledTitle: 'Missing', + notInstalledBody: 'Body', + installHint: 'Hint', + install: 'Install on host', + installing: 'Installing', + installCancelled: 'Cancelled', + installFailed: 'Failed', + noPackageManager: 'None' + } + }) +})) + +// eslint-disable-next-line no-restricted-imports +import { emitGatewayEvent } from '../../contrib/events' + +import { ScreenInstallCard } from './screen-install' + +const bot: RosterRow = { name: 'ops', sourceScoped: true, connectionId: 'host-a', connectionKind: 'remote' } + +const status: DisplayStatus = { + profile: 'ops', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: false, + missing: ['tigervnc'], + running: false, + pid: null, + display: null, + socket: null, + geometry: '1440x900', + install_command: 'sudo apt-get install -y tigervnc-standalone-server', + lease: { holder: 'agent', viewer_id: null, pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + calls.length = 0 +}) + +it('retains the bot socket before display.install and releases it when the done event lands', async () => { + const onInstalled = vi.fn() + const view = render() + + await act(async () => { + fireEvent.click(view.getByText('Install on host')) + }) + expect(calls).toEqual(['retain', 'request:display.install']) + + act(() => + emitGatewayEvent({ + type: 'display.install.done', + connectionId: 'host-a', + profile: 'ops', + payload: { profile_key: status.profile_key, code: 0, status: { ...status, installed: true } } + }) + ) + expect(calls).toEqual(['retain', 'request:display.install', 'release']) + expect(onInstalled).toHaveBeenCalledTimes(1) + view.unmount() + // Unmount after done must not double-release. + expect(calls.filter(call => call === 'release')).toHaveLength(1) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx index d69498ba5a3c..f5509527c5fb 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx @@ -14,7 +14,7 @@ import type { RpcEvent } from '@hermes/plugin-sdk' import { useCallback, useEffect, useRef, useState } from 'react' import { useBots } from './i18n' -import { displayRequest, type DisplayStatus, isEventForBotScreen } from './screen-connection' +import { displayRequest, type DisplayStatus, isEventForBotScreen, retainBotScreen } from './screen-connection' import type { RosterRow } from './types' const LOG_KEEP = 200 @@ -31,6 +31,16 @@ export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCar const [log, setLog] = useState([]) const [error, setError] = useState(null) const logEnd = useRef(null) + // Keeps the bot's socket open from display.install until done/failed: the log + // and done events ride that socket, and the SDK closes an idle one otherwise. + const retention = useRef<(() => void) | null>(null) + + const releaseRetention = useCallback(() => { + retention.current?.() + retention.current = null + }, []) + + useEffect(() => releaseRetention, [releaseRetention]) useEffect(() => { logEnd.current?.scrollIntoView({ block: 'end' }) @@ -53,6 +63,8 @@ export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCar return } + releaseRetention() + if (payload.code === 0 && payload.status?.installed) { setPhase('idle') onInstalled(payload.status) @@ -66,7 +78,7 @@ export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCar offLog() offDone() } - }, [bot, onInstalled, status.profile_key, t.screen.installCancelled, t.screen.installFailed]) + }, [bot, onInstalled, releaseRetention, status.profile_key, t.screen.installCancelled, t.screen.installFailed]) const install = useCallback(async () => { setPhase('running') @@ -74,12 +86,14 @@ export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCar setError(null) try { + retention.current = await retainBotScreen(bot) await displayRequest(bot, 'display.install') } catch (err) { + releaseRetention() setPhase('failed') setError(err instanceof Error ? err.message : String(err)) } - }, [bot]) + }, [bot, releaseRetention]) return (
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx index bfab9a0d38d7..75fc46120f95 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx @@ -6,6 +6,7 @@ import type * as ScreenConnection from './screen-connection' import type { RosterRow } from './types' const sockets = vi.hoisted(() => [] as Array<{ closeCodes: number[]; closed: boolean; close: (code?: number) => void }>) +const retention = vi.hoisted(() => ({ held: 0 })) vi.mock('@hermes/plugin-sdk', async () => { const { useStore } = await import('@nanostores/react') @@ -19,9 +20,21 @@ vi.mock('@hermes/plugin-sdk', async () => { GlyphSpinner: () => null, EmptyState: () => null, useValue: useStore, - host: { onEvent: onGatewayEvent } + host: { + onEvent: onGatewayEvent, + retainProfile: async () => { + retention.held += 1 + + return () => { + retention.held -= 1 + } + } + } } }) +vi.mock('./routing', () => ({ + botConnectionRoute: () => ({ connectionId: 'host-a', profile: 'default', targetProfile: 'default' }) +})) vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name })) vi.mock('./i18n', () => ({ useBots: () => ({ @@ -85,6 +98,7 @@ const status: DisplayStatus = { beforeEach(() => { $screenState.set({}) sockets.length = 0 + retention.held = 0 vi.mocked(displayRequest) .mockReset() .mockResolvedValue({ ...status, ticket: 'test-ticket', viewer_id: 'this-viewer' }) @@ -119,6 +133,18 @@ it('sends an intentional close before noVNC can send its statusless close on pan expect(sockets[0].closeCodes).toEqual([1000]) }) +it('pins the bot socket for the attach lifetime and lets go on unmount', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + expect(retention.held).toBe(1) + fireEvent.click(view.getByTitle('Reconnect')) + await waitFor(() => expect(sockets).toHaveLength(2)) + expect(retention.held).toBe(1) + view.unmount() + expect(retention.held).toBe(0) +}) + it('does not hand back while replacing a stream to reconnect the same viewer', async () => { const view = render() await waitFor(() => expect(sockets).toHaveLength(1)) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index bfd9ce0e44f9..fde4957389d7 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -15,7 +15,7 @@ import type { RpcEvent } from '@hermes/plugin-sdk' import { useCallback, useEffect, useRef, useState } from 'react' import { useBots } from './i18n' -import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isEventForBotScreen, leaseHeldBy, resolveScreenWsUrl, viewerHash } from './screen-connection' +import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isEventForBotScreen, leaseHeldBy, resolveScreenWsUrl, retainBotScreen, viewerHash } from './screen-connection' import { ScreenInstallCard } from './screen-install' import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenViewer } from './screen-state' import type { RosterRow } from './types' @@ -54,6 +54,9 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { const canvasHost = useRef(null) const rfb = useRef(null) const socket = useRef(null) + // Pins the bot's pooled gateway socket for the attach lifetime so display.lease + // events keep arriving for an inactive registry-routed bot. + const retention = useRef<(() => void) | null>(null) const [conn, setConn] = useState('idle') const [error, setError] = useState(null) const [busy, setBusy] = useState(false) @@ -94,6 +97,8 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { rfb.current = null socket.current?.close() socket.current = null + retention.current?.() + retention.current = null }, []) const attach = useCallback(async () => { @@ -110,6 +115,15 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { // Load the client BEFORE dialing: noVNC's Websock installs its own `onopen`, so a socket that // opened while the dynamic import was still in flight never hands it the open event. const Rfb = await loadRfb() + const retain = await retainBotScreen(bot) + + if (generation !== attachGeneration.current) { + retain() + + return + } + + retention.current = retain const observe = await displayRequest(bot, 'display.observe') const minted = { id: observe.viewer_id, hash: await viewerHash(observe.viewer_id) } setScreenStatus(bot, observe) From a0e78292b21853011815defde054b600905de9d5 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:28:26 -0700 Subject: [PATCH 46/55] fix(bot-screen): read the bridge's control-taken verdict from the raw close code noVNC 1.7's `disconnect` detail carries only `{clean}`; the pane was searching a `reason` that never arrives, so the "Another viewer took control" overlay was unreachable and a take-over looked like a clean stop. The pane now listens on the WebSocket it hands RFB (installed before RFB's own `onclose`) and treats close code 4000 as control-taken. (cherry picked from commit c0ed9a4cdb0d616cc727af26fd804f7e15a9f330) --- .../screen-pane-lifecycle.test.tsx | 52 +++++++++++++++++-- .../src/plugins/hermes-bots/screen-pane.tsx | 12 ++++- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx index 75fc46120f95..95eaa5952298 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx @@ -5,7 +5,11 @@ import type { DisplayStatus } from './screen-connection' import type * as ScreenConnection from './screen-connection' import type { RosterRow } from './types' -const sockets = vi.hoisted(() => [] as Array<{ closeCodes: number[]; closed: boolean; close: (code?: number) => void }>) +const sockets = vi.hoisted( + () => [] as Array<{ closeCodes: number[]; closed: boolean; close: (code?: number) => void; serverClose: (code: number) => void }> +) + +const rfbs = vi.hoisted(() => [] as Array<{ emit: (type: string, detail?: unknown) => void }>) const retention = vi.hoisted(() => ({ held: 0 })) vi.mock('@hermes/plugin-sdk', async () => { @@ -40,6 +44,7 @@ vi.mock('./i18n', () => ({ useBots: () => ({ screen: { title: 'Screen', + controlTaken: 'Another viewer took control', youControl: 'You control', handBack: 'Hand back', takeOver: 'Take over', @@ -57,13 +62,23 @@ vi.mock('./screen-connection', async importActual => ({ })) vi.mock('@novnc/novnc', () => ({ default: class { + private listeners = new Map void>>() constructor( _target: HTMLElement, private socket: { close: () => void } - ) {} - addEventListener(type: string, callback: () => void) { + ) { + rfbs.push(this) + } + emit(type: string, detail?: unknown) { + for (const listener of this.listeners.get(type) ?? []) { + listener({ detail }) + } + } + addEventListener(type: string, callback: (event: { detail?: unknown }) => void) { + this.listeners.set(type, [...(this.listeners.get(type) ?? []), callback]) + if (type === 'connect') { - queueMicrotask(callback) + queueMicrotask(() => callback({})) } } // noVNC 1.7 disconnects its WebSocket without a close code. @@ -98,6 +113,7 @@ const status: DisplayStatus = { beforeEach(() => { $screenState.set({}) sockets.length = 0 + rfbs.length = 0 retention.held = 0 vi.mocked(displayRequest) .mockReset() @@ -107,9 +123,24 @@ beforeEach(() => { class { closeCodes: number[] = [] closed = false + private onClose: Array<(event: { code: number }) => void> = [] constructor() { sockets.push(this) } + addEventListener(type: string, listener: (event: { code: number }) => void) { + if (type === 'close') { + this.onClose.push(listener) + } + } + // The bridge closing us: the raw close frame reaches our listener, then noVNC + // reports a statusless `disconnect` — the code is only on the socket event. + serverClose(code: number) { + this.closed = true + + for (const listener of this.onClose) { + listener({ code }) + } + } close(code?: number) { // Subsequent close calls cannot replace the frame already sent to the server. if (this.closed) { @@ -155,3 +186,16 @@ it('does not hand back while replacing a stream to reconnect the same viewer', a expect(sockets[1].closed).toBe(false) view.unmount() }) + +it('shows the control-taken overlay from the bridge close code, which noVNC does not forward', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + + act(() => { + sockets[0].serverClose(4000) + rfbs[0].emit('disconnect', { clean: true }) + }) + expect(view.getByText('Another viewer took control')).toBeTruthy() + view.unmount() +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index fde4957389d7..d9f3952637e0 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -34,6 +34,9 @@ type RfbLike = { type ConnState = 'idle' | 'attaching' | 'live' | 'control-taken' | 'error' +/** Bridge close code when another viewer took the lease (mirrors tui_gateway display bridge). */ +const CLOSE_CONTROL_TAKEN = 4000 + async function loadRfb(): Promise) => RfbLike> { const mod = (await import('@novnc/novnc')) as unknown as { default: new (...args: never[]) => RfbLike } @@ -137,6 +140,13 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { const ws = new WebSocket(url) ws.binaryType = 'arraybuffer' socket.current = ws + // noVNC 1.7's `disconnect` detail carries only {clean}; the bridge's verdict lives in + // the raw close frame (4000 = control-taken). Listen here, before RFB installs its + // own `onclose`, so the code is known by the time the disconnect event fires. + let closeCode = 0 + ws.addEventListener('close', event => { + closeCode = event.code + }) const client = new Rfb(canvasHost.current, ws, { shared: true }) client.scaleViewport = true client.resizeSession = false @@ -162,7 +172,7 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { const reason = event.detail?.reason ?? '' - if (reason.includes('control-taken')) { + if (closeCode === CLOSE_CONTROL_TAKEN || reason.includes('control-taken')) { setConn('control-taken') } else if (event.detail?.clean) { setConn('idle') From 2100432ca3f4c0364e8f6ff4b75e69bd7b384713 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:31:53 -0700 Subject: [PATCH 47/55] fix(bot-screen): older backends without display.* settle instead of checking forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bot whose Hermes predates the display RPCs answered `display.status` with JSON-RPC -32601; the portal swallowed it and stayed on "Checking the screen…" for good, and the pane retried on every mount. `isDisplayUnavailable` (same shape the session-control store recognises) marks the bot `unavailable` in $screenState: the sidebar portal and hero render nothing, the routines-page portal and the pane show "Update the bot's Hermes to use Screen" (screen.portalUnavailable / screen.unavailableTitle, all locales), and no further status probes are issued. (cherry picked from commit 9ff358422b60e735ce2846f1e30427e15ae767fd) --- apps/desktop/src/plugins/hermes-bots/i18n.ts | 10 ++++++ .../plugins/hermes-bots/screen-connection.ts | 13 ++++++++ .../src/plugins/hermes-bots/screen-hero.tsx | 3 +- .../hermes-bots/screen-isolation.test.tsx | 17 ++++++++++ .../src/plugins/hermes-bots/screen-pane.tsx | 12 +++++-- .../src/plugins/hermes-bots/screen-portal.tsx | 31 +++++++++++++------ .../src/plugins/hermes-bots/screen-state.ts | 15 +++++++++ 7 files changed, 88 insertions(+), 13 deletions(-) diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index 2d1786b18d91..22ebec4f6925 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -253,6 +253,8 @@ type BotsMessages = { portalStopped: string portalNotInstalled: string portalUnsupported: string + portalUnavailable: string + unavailableTitle: string recheck: string stoppedTitle: string stoppedBody: string @@ -521,6 +523,8 @@ const en: BotsMessages = { portalStopped: 'Stopped', portalNotInstalled: 'Not installed on host', portalUnsupported: 'Not available on this host', + portalUnavailable: 'Update the bot\u2019s Hermes to use Screen', + unavailableTitle: 'Screen needs a newer Hermes', recheck: 'Check again', stoppedTitle: 'Screen is off', stoppedBody: 'Start this bot\u2019s desktop to watch what it does and take over when it needs you.', @@ -784,6 +788,8 @@ const ja: BotsMessages = { portalStopped: '停止中', portalNotInstalled: 'ホストに未インストール', portalUnsupported: 'このホストでは利用できません', + portalUnavailable: 'Screen を使うにはボットの Hermes を更新してください', + unavailableTitle: 'Screen には新しい Hermes が必要です', recheck: '再確認', stoppedTitle: '画面はオフです', stoppedBody: 'このボットのデスクトップを起動すると、動作を見守り、必要なときに操作を引き継げます。', @@ -1042,6 +1048,8 @@ const zh: BotsMessages = { portalStopped: '已停止', portalNotInstalled: '主机未安装', portalUnsupported: '此主机不可用', + portalUnavailable: '更新机器人的 Hermes 以使用屏幕', + unavailableTitle: '屏幕需要更新版的 Hermes', recheck: '重新检查', stoppedTitle: '屏幕已关闭', stoppedBody: '启动此机器人的桌面，观看它的操作，并在需要时接管。', @@ -1300,6 +1308,8 @@ const zhHant: BotsMessages = { portalStopped: '已停止', portalNotInstalled: '主機未安裝', portalUnsupported: '此主機不可用', + portalUnavailable: '更新機器人的 Hermes 以使用螢幕', + unavailableTitle: '螢幕需要較新版的 Hermes', recheck: '重新檢查', stoppedTitle: '螢幕已關閉', stoppedBody: '啟動此機器人的桌面，觀看它的操作，並在需要時接手。', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts index e77ffea9b801..40b580358a98 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -82,6 +82,19 @@ export function leaseHeldBy(lease: DisplayLease | null | undefined, viewer: Scre return lease.viewer_id != null ? lease.viewer_id === viewer.id : lease.viewer_hash === viewer.hash } +/** JSON-RPC method-not-found: the bot's Hermes predates the `display.*` surface. */ +export function isDisplayUnavailable(error: unknown): boolean { + const record = typeof error === 'object' && error !== null ? (error as { code?: unknown; message?: unknown }) : null + + if (record?.code === -32601) { + return true + } + + const message = typeof record?.message === 'string' ? record.message.toLowerCase() : '' + + return message.includes('method not found') || message.includes('method-not-found') +} + /** Bare-profile fallback so a v1 local bot (no registry route) still resolves. */ export function botScreenRoute(bot: RosterRow): PluginProfileRoute | string { return botConnectionRoute(bot) ?? bot.name diff --git a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx index 93ad3e1bddc0..647b866d4e55 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx @@ -116,7 +116,7 @@ function ScreenHeroContent({ bot, meta }: { bot: RosterRow; meta?: BotMeta | nul const running = tone === 'live' || tone === 'human' || tone === 'other' const { dataUrl, boxRef, stale, suppressed } = useLiveThumbnail(bot, running) - if (tone === 'unsupported') { + if (tone === 'unsupported' || tone === 'unavailable') { return null } @@ -127,6 +127,7 @@ function ScreenHeroContent({ bot, meta }: { bot: RosterRow; meta?: BotMeta | nul off: t.screen.heroStopped, missing: t.screen.heroNotInstalled, unsupported: t.screen.portalUnsupported, + unavailable: t.screen.portalUnavailable, unknown: t.screen.heroConnecting }[tone] diff --git a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx index 926a34d8edd3..ffb835051902 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx @@ -33,6 +33,7 @@ vi.mock('./i18n', () => ({ heroOpenLive: 'Open live', heroStale: 'Last seen', heroSuppressed: 'Hidden while someone has control', + portalUnavailable: 'Update the bot', heroConnecting: 'Connecting' } }) @@ -182,3 +183,19 @@ it('captions a suppressed thumbnail as hidden-while-controlled and never ages it expect(view.getByRole('button').getAttribute('aria-label')).not.toContain('Last seen') view.unmount() }) + +it('settles on an older backend without display.*: portal tone is unavailable and the hero renders nothing', async () => { + vi.mocked(host.requestProfile).mockRejectedValue(Object.assign(new Error('Method not found: display.status'), { code: -32601 })) + const hook = renderHook(() => useScreenPortalState(botA)) + await act(async () => {}) + expect(hook.result.current.tone).toBe('unavailable') + hook.rerender() + await act(async () => {}) + expect(vi.mocked(host.requestProfile)).toHaveBeenCalledTimes(1) + hook.unmount() + + const view = render() + await act(async () => {}) + expect(view.container.firstChild).toBeNull() + view.unmount() +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index d9f3952637e0..9376a1a78610 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -15,9 +15,9 @@ import type { RpcEvent } from '@hermes/plugin-sdk' import { useCallback, useEffect, useRef, useState } from 'react' import { useBots } from './i18n' -import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isEventForBotScreen, leaseHeldBy, resolveScreenWsUrl, retainBotScreen, viewerHash } from './screen-connection' +import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isDisplayUnavailable, isEventForBotScreen, leaseHeldBy, resolveScreenWsUrl, retainBotScreen, viewerHash } from './screen-connection' import { ScreenInstallCard } from './screen-install' -import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenViewer } from './screen-state' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenUnavailable, setScreenViewer } from './screen-state' import type { RosterRow } from './types' type RfbLike = { @@ -71,6 +71,10 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { setScreenStatus(bot, next) setError(null) } catch (err) { + if (isDisplayUnavailable(err)) { + setScreenUnavailable(bot) + } + setError(err instanceof Error ? err.message : String(err)) } }, [bot]) @@ -256,6 +260,10 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { } }, [bot, viewer?.id]) + if (state?.unavailable) { + return + } + if (status && !status.supported) { return } diff --git a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx index 2587f284e530..5a050a2abc0c 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx @@ -17,15 +17,19 @@ import { useEffect } from 'react' import { $lastRoster } from './data' import { useBots } from './i18n' import { resolveBotConnectionRoute } from './routing' -import { type DisplayLease, displayRequest, type DisplayStatus, isEventForBotScreen, leaseHeldBy, type ScreenViewer } from './screen-connection' +import { type DisplayLease, displayRequest, type DisplayStatus, isDisplayUnavailable, isEventForBotScreen, leaseHeldBy, type ScreenViewer } from './screen-connection' import { openBotScreen } from './screen-open' -import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenUnavailable } from './screen-state' import type { BotMeta, RosterRow } from './types' -export type PortalTone = 'live' | 'human' | 'other' | 'off' | 'missing' | 'unsupported' | 'unknown' +export type PortalTone = 'live' | 'human' | 'other' | 'off' | 'missing' | 'unsupported' | 'unavailable' | 'unknown' /** Pure: map cached status + lease (+ this window's minted viewer, if attached) to what the portal says. */ -export function portalTone(status: DisplayStatus | null, lease: DisplayLease | null, viewer: ScreenViewer | null = null): PortalTone { +export function portalTone(status: DisplayStatus | null, lease: DisplayLease | null, viewer: ScreenViewer | null = null, unavailable = false): PortalTone { + if (unavailable) { + return 'unavailable' + } + if (!status) { return 'unknown' } @@ -56,6 +60,7 @@ const TONE_ICON: Record = { off: 'debug-stop', missing: 'cloud-download', unsupported: 'circle-slash', + unavailable: 'circle-slash', unknown: 'device-desktop' } @@ -66,6 +71,7 @@ const TONE_DOT: Record = { off: 'bg-(--ui-text-quaternary)', missing: 'bg-(--ui-text-quaternary)', unsupported: 'bg-(--ui-text-quaternary)', + unavailable: 'bg-(--ui-text-quaternary)', unknown: 'bg-(--ui-text-quaternary)' } @@ -76,7 +82,7 @@ export function useScreenPortalState(bot: RosterRow) { const profileKey = status?.profile_key useEffect(() => { - if (status) { + if (status || state?.unavailable) { return } @@ -88,14 +94,18 @@ export function useScreenPortalState(bot: RosterRow) { setScreenStatus(bot, next) } }) - .catch(() => { - /* offline bot / older backend: the portal stays in its unknown state */ + .catch((error: unknown) => { + // An older Hermes without display.* is a settled answer (hide the surface); + // an offline bot is transient and stays in its unknown state. + if (!cancelled && isDisplayUnavailable(error)) { + setScreenUnavailable(bot) + } }) return () => { cancelled = true } - }, [bot, status]) + }, [bot, state?.unavailable, status]) useEffect( () => @@ -109,7 +119,7 @@ export function useScreenPortalState(bot: RosterRow) { [bot, profileKey] ) - return { status, lease: state?.lease ?? null, tone: portalTone(status, state?.lease ?? null, state?.viewer ?? null) } + return { status, lease: state?.lease ?? null, tone: portalTone(status, state?.lease ?? null, state?.viewer ?? null, state?.unavailable) } } export function ScreenPortal({ bot, meta, compact = false }: { bot: RosterRow; meta?: BotMeta | null; compact?: boolean }) { @@ -123,10 +133,11 @@ export function ScreenPortal({ bot, meta, compact = false }: { bot: RosterRow; m off: t.screen.portalStopped, missing: t.screen.portalNotInstalled, unsupported: t.screen.portalUnsupported, + unavailable: t.screen.portalUnavailable, unknown: status?.display ?? '' }[tone] - if (tone === 'unsupported' && compact) { + if ((tone === 'unsupported' || tone === 'unavailable') && compact) { return null } diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.ts index 71e9ebfcc8af..8697e8dc72e1 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-state.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.ts @@ -15,6 +15,8 @@ export interface BotScreenState { lease: DisplayLease | null /** This window's server-minted identity for the bot's current attach; null until the pane observes. */ viewer: ScreenViewer | null + /** The bot's Hermes has no `display.*` methods (older backend): nothing to check, ever. */ + unavailable?: boolean } export const $screenState = atom>({}) @@ -30,6 +32,19 @@ export function setScreenStatus(bot: RosterRow, status: DisplayStatus): void { $screenState.set({ ...current, [key]: { status, lease: status.lease ?? prev?.lease ?? null, viewer: prev?.viewer ?? null } }) } +/** `display.status` answered method-not-found: remember it so no surface keeps "checking". */ +export function setScreenUnavailable(bot: RosterRow): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + + if (prev?.unavailable) { + return + } + + $screenState.set({ ...current, [key]: { status: null, lease: null, viewer: null, unavailable: true } }) +} + export function setScreenLease(bot: RosterRow, lease: DisplayLease): void { const key = botSelectionKey(bot) const current = $screenState.get() From 562331f1864eaf76e43c8b28eafe2c5de14a74d0 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:32:56 -0700 Subject: [PATCH 48/55] fix(bot-screen): a slower display.status reply can no longer roll back a newer lease Lease events and status replies race on the wire; the cache took whichever landed last, so a status reply describing the pre-take-over lease could flip the pane back to "Bot is in control" while a human held it. DisplayLease keeps the backend's monotonic `epoch`; both setScreenStatus and setScreenLease drop a lease whose epoch is below the stored one. Payloads without an epoch (older backends) are applied as before. (cherry picked from commit 37f8b91067f8af8f11127249f6f689547238d8af) --- .../plugins/hermes-bots/screen-connection.ts | 2 + .../plugins/hermes-bots/screen-state.test.ts | 61 +++++++++++++++++++ .../src/plugins/hermes-bots/screen-state.ts | 14 ++++- 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/src/plugins/hermes-bots/screen-state.test.ts diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts index 40b580358a98..671fac03dd7e 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -25,6 +25,8 @@ export interface DisplayLease { since: number reason: string pending_handoff: null | string + /** Monotonic per transition; a lower epoch is an older snapshot, never newer truth. */ + epoch?: number } export interface DisplayStatus { diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts new file mode 100644 index 000000000000..6e294921852f --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts @@ -0,0 +1,61 @@ +/** + * Lease ordering: `display.lease` events and `display.status` replies race on + * the wire. A slower status reply describing an OLDER lease must never roll + * back the newer event — the backend's monotonic `epoch` is the tiebreak. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { DisplayLease, DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name })) + +import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' + +const bot: RosterRow = { name: 'ops' } + +const agent: DisplayLease = { holder: 'agent', viewer_id: null, viewer_hash: null, since: 1, reason: '', pending_handoff: null, epoch: 3 } +const human: DisplayLease = { ...agent, holder: 'human', viewer_hash: 'abc123abc123', epoch: 4 } + +const statusWith = (lease: DisplayLease): DisplayStatus => ({ + profile: 'ops', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 1, + display: ':20', + socket: null, + geometry: '1440x900', + install_command: null, + lease +}) + +beforeEach(() => $screenState.set({})) + +describe('lease epoch ordering', () => { + it('a status reply carrying an older epoch does not roll back a newer lease event', () => { + setScreenLease(bot, human) + setScreenStatus(bot, statusWith(agent)) + + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(human) + // The status itself still lands — only its stale lease is ignored. + expect(screenStateFor($screenState.get(), bot)?.status?.running).toBe(true) + }) + + it('a lease event with an older epoch is ignored; a newer or epoch-less one applies', () => { + setScreenLease(bot, human) + setScreenLease(bot, agent) + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(human) + + const released = { ...agent, epoch: 5 } + setScreenLease(bot, released) + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(released) + + const legacy = { ...human, epoch: undefined } + setScreenLease(bot, legacy) + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(legacy) + }) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.ts index 8697e8dc72e1..06bfb8e6f8dd 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-state.ts +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.ts @@ -25,11 +25,19 @@ export function screenStateFor(all: Record, bot: RosterR return all[botSelectionKey(bot)] ?? null } +/** A lease whose epoch is below the one we hold is a slower response about the + * past (a `display.status` reply overtaken by a `display.lease` event). Payloads + * without an epoch — older backends — are always applied. */ +function isOlderLease(prev: DisplayLease | null | undefined, next: DisplayLease): boolean { + return typeof next.epoch === 'number' && typeof prev?.epoch === 'number' && next.epoch < prev.epoch +} + export function setScreenStatus(bot: RosterRow, status: DisplayStatus): void { const key = botSelectionKey(bot) const current = $screenState.get() const prev = current[key] - $screenState.set({ ...current, [key]: { status, lease: status.lease ?? prev?.lease ?? null, viewer: prev?.viewer ?? null } }) + const lease = status.lease && !isOlderLease(prev?.lease, status.lease) ? status.lease : (prev?.lease ?? null) + $screenState.set({ ...current, [key]: { status, lease, viewer: prev?.viewer ?? null } }) } /** `display.status` answered method-not-found: remember it so no surface keeps "checking". */ @@ -50,6 +58,10 @@ export function setScreenLease(bot: RosterRow, lease: DisplayLease): void { const current = $screenState.get() const prev = current[key] + if (isOlderLease(prev?.lease, lease)) { + return + } + if ( prev?.lease && prev.lease.holder === lease.holder && From a659f6103897f41832796c4cc093dc99ff1038b8 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:37:10 -0700 Subject: [PATCH 49/55] feat(bot-screen): "Hand back (force)" for a lease this window no longer owns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a Desktop reload the pane attaches under a fresh server-minted viewer id while the old one still holds the lease, so the docs' "click Hand back" had no button to click — only Take over. When holder is human and this window is not the holder, the pane now also offers "Hand back (force)", which calls display.lease.release with {force: true} (the server refuses a plain release from a non-holder). i18n cleanup: drop the unused screen.recheck key, move the hardcoded "Update Hermes Desktop…" notice and the tab title suffix in screen-open.tsx into the bundle (screen.openNeedsUpdate / screen.title). All four locales. (cherry picked from commit 5cf98770f72da261a5471467170993833d15118b) --- apps/desktop/src/plugins/hermes-bots/i18n.ts | 20 ++++++--- .../src/plugins/hermes-bots/screen-open.tsx | 5 ++- .../src/plugins/hermes-bots/screen-pane.tsx | 44 ++++++++++++------- .../screen-viewer-identity.test.tsx | 18 ++++++++ 4 files changed, 65 insertions(+), 22 deletions(-) diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index 22ebec4f6925..fd860551e2d5 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -255,7 +255,6 @@ type BotsMessages = { portalUnsupported: string portalUnavailable: string unavailableTitle: string - recheck: string stoppedTitle: string stoppedBody: string start: string @@ -264,6 +263,9 @@ type BotsMessages = { reconnect: string takeOver: string handBack: string + handBackForce: string + handBackForceHint: string + openNeedsUpdate: string youControl: string otherControls: string agentControls: string @@ -525,7 +527,6 @@ const en: BotsMessages = { portalUnsupported: 'Not available on this host', portalUnavailable: 'Update the bot\u2019s Hermes to use Screen', unavailableTitle: 'Screen needs a newer Hermes', - recheck: 'Check again', stoppedTitle: 'Screen is off', stoppedBody: 'Start this bot\u2019s desktop to watch what it does and take over when it needs you.', start: 'Start screen', @@ -534,6 +535,9 @@ const en: BotsMessages = { reconnect: 'Reconnect', takeOver: 'Take over', handBack: 'Hand back', + handBackForce: 'Hand back (force)', + handBackForceHint: 'Release a lease held by a viewer that is no longer here, e.g. after a reload.', + openNeedsUpdate: 'Update Hermes Desktop to open bot screens.', youControl: 'You are in control', otherControls: 'Another viewer is in control', agentControls: 'Bot is in control', @@ -790,7 +794,6 @@ const ja: BotsMessages = { portalUnsupported: 'このホストでは利用できません', portalUnavailable: 'Screen を使うにはボットの Hermes を更新してください', unavailableTitle: 'Screen には新しい Hermes が必要です', - recheck: '再確認', stoppedTitle: '画面はオフです', stoppedBody: 'このボットのデスクトップを起動すると、動作を見守り、必要なときに操作を引き継げます。', start: '画面を起動', @@ -799,6 +802,9 @@ const ja: BotsMessages = { reconnect: '再接続', takeOver: '引き継ぐ', handBack: '戻す', + handBackForce: '強制的に戻す', + handBackForceHint: 'もう存在しないビューア（再読み込み後など）が保持しているリースを解放します。', + openNeedsUpdate: 'ボットの画面を開くには Hermes Desktop を更新してください。', youControl: 'あなたが操作中', otherControls: '別のビューアが操作中', agentControls: 'ボットが操作中', @@ -1050,7 +1056,6 @@ const zh: BotsMessages = { portalUnsupported: '此主机不可用', portalUnavailable: '更新机器人的 Hermes 以使用屏幕', unavailableTitle: '屏幕需要更新版的 Hermes', - recheck: '重新检查', stoppedTitle: '屏幕已关闭', stoppedBody: '启动此机器人的桌面，观看它的操作，并在需要时接管。', start: '启动屏幕', @@ -1059,6 +1064,9 @@ const zh: BotsMessages = { reconnect: '重新连接', takeOver: '接管', handBack: '交还', + handBackForce: '强制交还', + handBackForceHint: '释放已不在场的查看者（例如重新加载后）持有的控制权。', + openNeedsUpdate: '更新 Hermes Desktop 以打开机器人屏幕。', youControl: '你正在控制', otherControls: '另一位查看者正在控制', agentControls: '机器人正在控制', @@ -1310,7 +1318,6 @@ const zhHant: BotsMessages = { portalUnsupported: '此主機不可用', portalUnavailable: '更新機器人的 Hermes 以使用螢幕', unavailableTitle: '螢幕需要較新版的 Hermes', - recheck: '重新檢查', stoppedTitle: '螢幕已關閉', stoppedBody: '啟動此機器人的桌面，觀看它的操作，並在需要時接手。', start: '啟動螢幕', @@ -1319,6 +1326,9 @@ const zhHant: BotsMessages = { reconnect: '重新連線', takeOver: '接手', handBack: '交還', + handBackForce: '強制交還', + handBackForceHint: '釋放已不在場的檢視者（例如重新載入後）持有的控制權。', + openNeedsUpdate: '更新 Hermes Desktop 以開啟機器人螢幕。', youControl: '你正在控制', otherControls: '另一位檢視者正在控制', agentControls: '機器人正在控制', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-open.tsx b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx index ae0ff946c64f..9d05539c4d88 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-open.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx @@ -6,6 +6,7 @@ import { host } from '@hermes/plugin-sdk' import { botSelectionKey } from './data' +import { botsText } from './i18n' import { displayName } from './labels' import { BotScreenPane } from './screen-pane' import { ID } from './shared' @@ -19,7 +20,7 @@ export function screenPaneId(bot: RosterRow): string { export function openBotScreen(bot: RosterRow, meta?: BotMeta | null): void { if (typeof host.openWorkspace !== 'function') { - host.notify({ kind: 'info', message: 'Update Hermes Desktop to open bot screens.' }) + host.notify({ kind: 'info', message: botsText().screen.openNeedsUpdate }) return } @@ -33,7 +34,7 @@ export function openBotScreen(bot: RosterRow, meta?: BotMeta | null): void { } const close = host.openWorkspace(`${ID}:screen:${key}`, { - title: `${displayName(bot, meta ?? null)} · Screen`, + title: `${displayName(bot, meta ?? null)} · ${botsText().screen.title}`, minWidth: '28rem', render: () => , onClose: () => { diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx index 9376a1a78610..c555e2fc3ea7 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -247,18 +247,25 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { } }, [attach, bot, conn, viewer?.id]) - const handBack = useCallback(async () => { - setBusy(true) - - try { - const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', { viewer_id: viewer?.id }) - setScreenLease(bot, result.lease) - } catch (err) { - setError(err instanceof Error ? err.message : String(err)) - } finally { - setBusy(false) - } - }, [bot, viewer?.id]) + // `force` is the escape hatch for a lease this window no longer owns (a reload + // minted a fresh viewer id; the old one still holds): the server refuses a + // plain release from anyone but the holder. + const handBack = useCallback( + async (force = false) => { + setBusy(true) + + try { + const params = force ? { force: true } : { viewer_id: viewer?.id } + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', params) + setScreenLease(bot, result.lease) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, + [bot, viewer?.id] + ) if (state?.unavailable) { return @@ -319,9 +326,16 @@ export function BotScreenPane({ bot }: { bot: RosterRow }) { {t.screen.handBack} ) : ( - + <> + {humanOther ? ( + + ) : null} + + )}
{/* data-terminal: the same keyboard-ownership marker the terminal pane uses, so the app's - type-to-focus / bare-key shortcuts never steal keystrokes meant for the remote screen. */} -
+ type-to-focus / bare-key shortcuts never steal keystrokes meant for the remote screen. + data-remote-screen: tells the ⌘W close-tab router this is NOT a local terminal tab — + the chord belongs to the remote desktop, nothing local should close. */} +
{conn === 'attaching' ? (
{t.screen.attaching} From 090cc95d2b99bba342b5fab57494421c2b5859dd Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:43:08 -0700 Subject: [PATCH 51/55] perf(bot-screen): sidebar group portal keeps one row identity across re-renders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProfileGroupScreenPortal rebuilt its synthesized RosterRow on every render, and the portal's effects (status probe, display.lease subscription) key on that row — so every sidebar paint re-subscribed the lease listener and re-ran the probe guard. The row is now memoised on (connectionId, profile, roster). The other half of the remount lives outside this plugin: gateway-groups.tsx hands ContribRender an inline `() => render(route)` closure, and ContribRender mounts its argument AS a component type, so the whole contribution remounts on every group-header render. That needs a stable component per (item, route) in ProfileGroupHeaderSlot (useMemo/useCallback), not a plugin-side change. (cherry picked from commit c8a606f76d8fb69c21773de7024c698cfbe22732) --- .../hermes-bots/screen-isolation.test.tsx | 15 +++++++++- .../src/plugins/hermes-bots/screen-portal.tsx | 30 +++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx index ffb835051902..00b65c11e8f2 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx @@ -11,7 +11,7 @@ vi.mock('@hermes/plugin-sdk', async () => { Codicon: () => null, useValue: useStore, resolveSiblingWsUrl: vi.fn(), - host: { onEvent: onGatewayEvent, requestProfile: vi.fn() } + host: { onEvent: vi.fn(onGatewayEvent), requestProfile: vi.fn() } } }) vi.mock('./data', async () => { @@ -199,3 +199,16 @@ it('settles on an older backend without display.*: portal tone is unavailable an expect(view.container.firstChild).toBeNull() view.unmount() }) + +it('a sidebar group portal keeps its lease subscription across parent re-renders (no per-paint row rebuild)', async () => { + vi.mocked(host.requestProfile).mockResolvedValue(status) + const view = render() + await act(async () => {}) + const subscriptions = vi.mocked(host.onEvent).mock.calls.length + + view.rerender() + view.rerender() + await act(async () => {}) + expect(vi.mocked(host.onEvent).mock.calls.length).toBe(subscriptions) + view.unmount() +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx index 5a050a2abc0c..7afa011f2436 100644 --- a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx +++ b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx @@ -12,7 +12,7 @@ import { Codicon, host, useValue } from '@hermes/plugin-sdk' import type { RpcEvent } from '@hermes/plugin-sdk' import type { ProfileGroupRoute } from '@hermes/plugin-sdk' -import { useEffect } from 'react' +import { useEffect, useMemo } from 'react' import { $lastRoster } from './data' import { useBots } from './i18n' @@ -168,18 +168,24 @@ export function ScreenPortal({ bot, meta, compact = false }: { bot: RosterRow; m * roster filter still gets its portal (the portal only needs a routable row). */ export function ProfileGroupScreenPortal({ route }: { route: ProfileGroupRoute }) { const roster = useValue($lastRoster) + const { connectionId, profile } = route - const bot = - roster.find(row => { - const resolved = resolveBotConnectionRoute(row) - - return resolved.route - ? resolved.route.profile === route.profile && resolved.route.connectionId === (route.connectionId ?? 'local') - : row.name === route.profile && route.connectionId === null - }) ?? - (route.connectionId - ? ({ name: route.profile, sourceScoped: true, connectionId: route.connectionId, connectionKind: route.connectionId === 'local' ? 'local' : 'remote' } as RosterRow) - : ({ name: route.profile } as RosterRow)) + // Stable row identity: the portal's effects key on `bot`, so a synthesized row + // rebuilt every render would re-subscribe the lease listener on every sidebar paint. + const bot = useMemo( + () => + roster.find(row => { + const resolved = resolveBotConnectionRoute(row) + + return resolved.route + ? resolved.route.profile === profile && resolved.route.connectionId === (connectionId ?? 'local') + : row.name === profile && connectionId === null + }) ?? + (connectionId + ? ({ name: profile, sourceScoped: true, connectionId, connectionKind: connectionId === 'local' ? 'local' : 'remote' } as RosterRow) + : ({ name: profile } as RosterRow)), + [connectionId, profile, roster] + ) return } From 171b5a125d6a2ee5f05b37bdbb826bb41fe98e9b Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:48:02 -0700 Subject: [PATCH 52/55] fix(bot-screen): wire lane call sites: atomic install claim, redacted lease views everywhere, backend rebind on display change, footgun annotations, grace in schema --- hermes_cli/config_defaults.py | 2 +- hermes_cli/pty_bridge.py | 4 ++-- scripts/profile-tui.py | 2 +- tools/computer_use/schema.py | 1 + tools/computer_use/tool.py | 14 ++++++++++---- tui_gateway/methods_display.py | 8 ++++---- tui_gateway/methods_display_watch.py | 5 +++-- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index ac8e5b69e7f9..6216261a534d 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2243,7 +2243,7 @@ def _aux(timeout, *, reasoning_effort=True, **extra): "paste_collapse_char_threshold": 2000, # Bot Desktop: a headless Xfce screen per profile on the gateway host (Linux), streamed to Hermes - # Desktop where a human can watch, take over (logins, 2FA, CAPTCHAs) and hand back. `hermes desktop`. + # Desktop where a human can watch, take over (logins, 2FA, CAPTCHAs) and hand back. `hermes computer-use screen`. "bot_desktop": { "geometry": "1440x900", # Opt-in: start the screen automatically the first time computer_use needs a display on a headless diff --git a/hermes_cli/pty_bridge.py b/hermes_cli/pty_bridge.py index 90d6a61a6c33..2788eb9ecee1 100644 --- a/hermes_cli/pty_bridge.py +++ b/hermes_cli/pty_bridge.py @@ -10,13 +10,13 @@ import asyncio import errno -import fcntl +import fcntl # windows-footgun: ok — POSIX-only module by design (see docstring) import os import select import signal import struct import sys -import termios +import termios # windows-footgun: ok — POSIX-only module by design (see docstring) import time from typing import Optional, Sequence diff --git a/scripts/profile-tui.py b/scripts/profile-tui.py index 4d86aa057bfa..a59b76659d82 100755 --- a/scripts/profile-tui.py +++ b/scripts/profile-tui.py @@ -26,7 +26,7 @@ import argparse import json import os -import pty +import pty # windows-footgun: ok — dev profiling script, POSIX pty by design import select import signal import sqlite3 diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index e984bfe3f60f..880d9bdc5827 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -153,6 +153,7 @@ ), }, "seconds": {"type": "number", "description": "wait: seconds to pause (max 30). wait_for_human: how long to block for the hand-back (default 600, max 1800)."}, + "grace": {"type": "number", "description": "wait_for_human: seconds to wait for someone to take over before returning no_takeover (default 60); once a human holds control the full `seconds` applies."}, "raise_window": { "type": "boolean", "description": ( diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 04b6715795c4..32f892db38c1 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -79,6 +79,7 @@ def _input_target_mismatch(backend, requested_app: str) -> Optional[str]: _backends: Dict[str, ComputerUseBackend] = {} _backend_call_locks: Dict[str, threading.RLock] = {} _backend_permission_modes: Dict[str, str] = {} +_backend_displays: Dict[str, str] = {} # DISPLAY the cached backend was spawned against (Bot Desktop rebind) # (home key, provider, model) → bool. The decision reads the active profile's config (auxiliary.vision # override, declared supports_vision), so a multiplexed process must not serve profile A's verdict to B. _AUX_VISION_ROUTE_CACHE: Dict[Tuple[str, str, str], bool] = {} @@ -133,7 +134,9 @@ def _install_backend(sid: str, backend: ComputerUseBackend, permission_mode: str """Record a backend in the session caches (the empty session also mirrors it onto the ``_backend`` hook). Caller holds ``_backend_lock``.""" global _backend + from tools.computer_use.cua_backend import desktop_identity _backends[sid], _backend_permission_modes[sid] = backend, permission_mode + _backend_displays[sid] = desktop_identity() _backend_call_locks[sid] = threading.RLock() _backend = backend if sid == "" else _backend return backend @@ -142,7 +145,7 @@ def _detach_locked(sid: str) -> Tuple[Optional[ComputerUseBackend], Optional[thr """Remove one session's cache entries, plus the ``_backend`` injection hook when it aliases the empty session (older callers/tests may populate only the hook). Caller holds ``_backend_lock``.""" global _backend - _backend_permission_modes.pop(sid, None) + _backend_permission_modes.pop(sid, None), _backend_displays.pop(sid, None) backend, call_lock = _backends.pop(sid, None), _backend_call_locks.pop(sid, None) if sid == "": backend = _backend if backend is None else backend @@ -170,9 +173,12 @@ def _get_backend(session_id: str = "") -> ComputerUseBackend: backend = _new_backend(permission_mode) backend.start() # under the cache lock: one backend per session; a concurrent toggle releases it return _install_backend(sid, backend, permission_mode) - if _backend_permission_modes.get(sid, "standard") == permission_mode: + from tools.computer_use.cua_backend import backend_display_stale, desktop_identity + if (_backend_permission_modes.get(sid, "standard") == permission_mode + and not backend_display_stale(_backend_displays.get(sid, ""), desktop_identity())): return cached - # Cua's mode is immutable after daemon startup: a /yolo toggle replaces only this session's backend. + # Cua's mode and DISPLAY are fixed at daemon startup: a /yolo toggle, or a Bot Desktop that started + # (or moved) after this backend was cached, replaces only this session's backend. _, stale_lock = _detach_locked(sid) # stopped outside the cache lock; the loop re-reads the mode first _stop_backend(cached, stale_lock, lambda e: None) @@ -207,7 +213,7 @@ def _shutdown_backend_atexit() -> None: if _backend is not None: unique.setdefault(id(_backend), (_backend, _backend_call_locks.get(""))) _backend = None - _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear() + _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear(), _backend_displays.clear() with _approval_lock: _session_auto_approve.clear(), _always_allow.clear(), _escalation_warned.clear() for backend, call_lock in unique.values(): diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py index 38e4fba69762..660a8a685ff1 100644 --- a/tui_gateway/methods_display.py +++ b/tui_gateway/methods_display.py @@ -177,7 +177,7 @@ def _line(text: str) -> None: def _run() -> None: try: - code = _bd_install.install_packages(ask_password=_ask_password, on_line=_line) + code = _bd_install.install_packages(ask_password=_ask_password, on_line=_line, claimed=True) except Exception as e: _line(f"install failed: {e}") code = 1 @@ -185,7 +185,7 @@ def _run() -> None: "status": _display_snapshot()}) try: - _bd_install.assert_not_running() + _bd_install.claim() # atomic: two fast clicks cannot both start a package manager except _bd_install.InstallBusy as e: return _err(rid, _DISPLAY_ERR, str(e)) threading.Thread(target=ctx.run, args=(_run,), name=f"bot-desktop-install:{profile_key}", daemon=True).start() @@ -200,7 +200,7 @@ def _(rid, params: dict) -> dict: if not viewer_id: return _err(rid, _DISPLAY_ERR, "viewer_id required") lease = _bd_lease.acquire(viewer_id, reason=str(params.get("reason") or "")) - return _ok(rid, {"lease": lease.as_dict()}) + return _ok(rid, {"lease": _lease_view(lease)}) @method("display.lease.release") @@ -214,7 +214,7 @@ def _(rid, params: dict) -> dict: return _err(rid, _DISPLAY_ERR, "viewer_id required to release another viewer's lease (or pass force: true)", data={"code": "viewer_mismatch"}) lease = _bd_lease.release(viewer_id) - return _ok(rid, {"lease": lease.as_dict()}) + return _ok(rid, {"lease": _lease_view(lease)}) def register(server) -> None: diff --git a/tui_gateway/methods_display_watch.py b/tui_gateway/methods_display_watch.py index 7c0efb162210..3050d08b7678 100644 --- a/tui_gateway/methods_display_watch.py +++ b/tui_gateway/methods_display_watch.py @@ -26,8 +26,9 @@ def _lease_event_payload(profile_key: str, lease) -> dict: - # Mirrors methods_display._install_lease_listener's nested payload; keep the two in step. - return {"profile_key": profile_key, "lease": lease.as_dict()} + # Same shape and the same redaction (viewer_hash, never the raw id) as the in-process broadcast. + from tui_gateway.methods_display import _lease_view + return {"profile_key": profile_key, "lease": _lease_view(lease)} def _watched_lease_homes() -> list[Path]: From b9887ea6dc3fb3c261e5d9e5dd28850bce04039d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:53:15 -0700 Subject: [PATCH 53/55] =?UTF-8?q?fix(desktop):=20=E2=8C=98W=20on=20a=20rem?= =?UTF-8?q?ote=20bot=20screen=20closes=20nothing=20local;=20profile-group?= =?UTF-8?q?=20header=20rows=20keep=20one=20render=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/desktop/src/app/chat/close-tab.test.ts | 20 +++++++++++++++++ apps/desktop/src/app/chat/close-tab.ts | 6 +++++ .../src/app/chat/sidebar/gateway-groups.tsx | 22 +++++++++++++++---- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/app/chat/close-tab.test.ts b/apps/desktop/src/app/chat/close-tab.test.ts index 219f20df6d49..f0785a6bc982 100644 --- a/apps/desktop/src/app/chat/close-tab.test.ts +++ b/apps/desktop/src/app/chat/close-tab.test.ts @@ -7,6 +7,12 @@ const nextSessionTileForWorkspace = vi.fn<() => null | string>(() => null) const closeSessionTile = vi.fn() const requestFreshSession = vi.fn() +const closeActiveTerminal = vi.fn() + +vi.mock('@/app/right-sidebar/terminal/terminals', () => ({ + closeActiveTerminal: () => closeActiveTerminal() +})) + vi.mock('@/components/pane-shell/tree/store', () => ({ closeFocusedSessionTab: () => closeFocusedSessionTab(), closeFocusedToolTab: () => closeFocusedToolTab() @@ -136,6 +142,20 @@ describe('closeWorkspaceTab', () => { expect(requestFreshSession).toHaveBeenCalledTimes(1) }) + it('a focused remote bot screen swallows ⌘W: no terminal tab, no session tab closes', async () => { + loadedMainOnly() + const combo = await import('@/lib/keybinds/combo') + const spy = vi.spyOn(combo, 'isFocusWithin').mockImplementation(selector => selector === '[data-remote-screen]' || selector === '[data-terminal]') + + try { + expect(closeActiveTab(vi.fn())).toBe(true) + expect(closeActiveTerminal).not.toHaveBeenCalled() + expect(requestFreshSession).not.toHaveBeenCalled() + } finally { + spy.mockRestore() + } + }) + it('a focused tool panel (terminal / logs) claims ⌘W before main empties', () => { loadedMainOnly() closeFocusedToolTab.mockReturnValue(true) diff --git a/apps/desktop/src/app/chat/close-tab.ts b/apps/desktop/src/app/chat/close-tab.ts index 4304138904ff..6aa681fba227 100644 --- a/apps/desktop/src/app/chat/close-tab.ts +++ b/apps/desktop/src/app/chat/close-tab.ts @@ -67,6 +67,12 @@ export function closeWorkspaceTab(loadSessionIntoWorkspace?: (storedSessionId: s * with its own tab strip closes ITS tab instead of main's. */ export function closeActiveTab(loadSessionIntoWorkspace?: (storedSessionId: string) => void): boolean { + // A remote bot screen borrows the terminal's keyboard ownership marker so bare keys reach it; ⌘W + // there belongs to the remote desktop, never to a local terminal tab or the session tab behind it. + if (isFocusWithin('[data-remote-screen]')) { + return true + } + if (isFocusWithin('[data-terminal]')) { closeActiveTerminal() diff --git a/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx b/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx index 65504ebb8399..829c85ff0815 100644 --- a/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx +++ b/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx @@ -2,7 +2,7 @@ import type { useSensors } from '@dnd-kit/core' import { arrayMove } from '@dnd-kit/sortable' import { useStore } from '@nanostores/react' import type { ReactNode } from 'react' -import { useState } from 'react' +import { useMemo, useState } from 'react' import { type NewSessionSplitHandler, startNewSessionDrag } from '@/app/chat/new-session-drag' import { type ProfileGroupHeaderContribution, SIDEBAR_PROFILE_GROUP_HEADER_AREA } from '@/app/routes' @@ -339,14 +339,28 @@ function ProfileGroupHeaderSlot({ connectionId, profile }: { connectionId: null return null } - const render = data.render - return ( - render({ connectionId, profile })} /> + ) })}
) } + +/** One stable render identity per (render, connection, profile): ContribRender mounts whatever + * function it is handed, so an inline closure would remount the contribution on every paint. */ +function ProfileGroupHeaderItem({ + connectionId, + profile, + render +}: { + connectionId: null | string + profile: string + render: ProfileGroupHeaderContribution['render'] +}) { + const Row = useMemo(() => () => render({ connectionId, profile }), [connectionId, profile, render]) + + return +} From d65af42305c4b51227e51aafb17fe7f00ba4de50 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:03:52 -0700 Subject: [PATCH 54/55] test(bot-screen): fixtures follow the integrated contracts (provenance fence, claimed install slot) --- tests/tools/test_bot_desktop_browser.py | 2 +- tests/tui_gateway/test_display_methods.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tools/test_bot_desktop_browser.py b/tests/tools/test_bot_desktop_browser.py index 2cfdc5523b47..451702e2c4f3 100644 --- a/tests/tools/test_bot_desktop_browser.py +++ b/tests/tools/test_bot_desktop_browser.py @@ -80,7 +80,7 @@ def spawn(task_id, session_info, cmd_parts, *rest): monkeypatch.setattr(session, "_spawn_and_collect", spawn) monkeypatch.setattr(session._lp, "_lightpanda_fallback_reason", lambda *a: None) - info = {"session_name": "h_abc", "cdp_url": None} + info = {"session_name": "h_abc", "cdp_url": None, "features": {"local": True}} monkeypatch.setattr(browser, "running_instance_cdp_port", lambda d, **kw: 41234) session._run_browser_command_unfenced("t", "open", ["https://x"], 10, None, "agent-browser", info) diff --git a/tests/tui_gateway/test_display_methods.py b/tests/tui_gateway/test_display_methods.py index 67a6f207f081..4998946b47fa 100644 --- a/tests/tui_gateway/test_display_methods.py +++ b/tests/tui_gateway/test_display_methods.py @@ -24,7 +24,7 @@ def test_install_worker_keeps_the_requested_profile_scope(tmp_path, monkeypatch) seen = {} done = threading.Event() - def fake_install(*, ask_password, on_line, timeout_seconds=900.0): + def fake_install(*, ask_password, on_line, timeout_seconds=900.0, claimed=False): seen["home"] = str(get_hermes_home()) done.set() return 0 From d13f3425bb75acc9f99a1e8c2e40e8a42b98ee3a Mon Sep 17 00:00:00 2001 From: SmokeDev Date: Sat, 12 Sep 2026 18:25:01 -0700 Subject: [PATCH 55/55] test(computer-use): cover native Windows and macOS registry dispatch Signed-off-by: SmokeDev --- .../test_computer_use_native_platform.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/tools/test_computer_use_native_platform.py diff --git a/tests/tools/test_computer_use_native_platform.py b/tests/tools/test_computer_use_native_platform.py new file mode 100644 index 000000000000..4a018eb39fb9 --- /dev/null +++ b/tests/tools/test_computer_use_native_platform.py @@ -0,0 +1,38 @@ +"""Exercise existing computer-use actions on the actual Windows/macOS CI hosts.""" + +import json +import sys + +import pytest + +from tools.computer_use import tool + + +@pytest.fixture +def backend(monkeypatch): + tool.reset_backend_for_tests() + monkeypatch.setenv("HERMES_COMPUTER_USE_BACKEND", "noop") + value = tool._get_backend() + yield value + tool.reset_backend_for_tests() + + +@pytest.mark.parametrize("host_platform", [ + pytest.param("win32", marks=pytest.mark.windows_only), + pytest.param("darwin", marks=pytest.mark.macos_only), +]) +@pytest.mark.parametrize("args, expected_call", [ + ({"action": "capture", "mode": "ax"}, "capture"), + ({"action": "click", "coordinate": [10, 10]}, "click"), + ({"action": "type", "text": "test input"}, "type"), + ({"action": "list_windows"}, "list_windows"), +]) +def test_native_computer_use_dispatches_to_backend(backend, args, expected_call, host_platform): + import tools.computer_use_tool # noqa: F401 - register the real tool handler + from tools.registry import registry + + assert sys.platform == host_platform + result = registry.dispatch("computer_use", args) + + assert "error" not in json.loads(result) + assert [name for name, _ in backend.calls] == [expected_call]