diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts
index d6b96cf348bc1..fd860551e2d59 100644
--- a/apps/desktop/src/plugins/hermes-bots/i18n.ts
+++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts
@@ -223,6 +223,56 @@ type BotsMessages = {
noMcpServers: string
}
+ /** Bot Screen: the bot's headless desktop on the gateway host, live in a pane. */
+ screen: {
+ title: string
+ menu: string
+ unsupportedTitle: string
+ unsupportedBody: string
+ notInstalledTitle: string
+ notInstalledBody: string
+ installHint: string
+ install: string
+ installing: string
+ installCancelled: string
+ installFailed: string
+ noPackageManager: string
+ portalTitle: string
+ portalOpen: string
+ heroStopped: string
+ heroNotInstalled: string
+ heroConnecting: string
+ heroStale: string
+ heroSuppressed: string
+ heroOpenLive: string
+ heroInstall: string
+ heroStart: string
+ portalWatching: string
+ portalYouControl: string
+ portalOtherControls: string
+ portalStopped: string
+ portalNotInstalled: string
+ portalUnsupported: string
+ portalUnavailable: string
+ unavailableTitle: string
+ stoppedTitle: string
+ stoppedBody: string
+ start: string
+ attaching: string
+ streamLost: string
+ reconnect: string
+ takeOver: string
+ handBack: string
+ handBackForce: string
+ handBackForceHint: string
+ openNeedsUpdate: string
+ youControl: string
+ otherControls: string
+ agentControls: string
+ controlTaken: string
+ handoffRequested: string
+ }
+
/** Bot-scoped scheduled jobs. Generic scheduling chrome (weekday names,
* Daily/Hourly, the job verbs) resolves against core's `cron` section. */
cron: {
@@ -446,6 +496,54 @@ const en: BotsMessages = {
searchHub: 'Search the hub (community + well-known sources)…',
noMcpServers: 'No MCP servers configured or in the catalog.'
},
+ screen: {
+ title: 'Screen',
+ menu: 'Open Screen',
+ unsupportedTitle: 'No bot screen on this host',
+ unsupportedBody: 'Bot screens run on Linux gateway hosts. This bot uses the host\u2019s own display.',
+ notInstalledTitle: 'Screen packages missing',
+ notInstalledBody: 'The gateway host needs TigerVNC and the Xfce core to give this bot a screen. Run on the host:',
+ installHint: 'Runs on the gateway host as the user Hermes runs as; sudo is asked for once, through Hermes.',
+ install: 'Install on host',
+ installing: 'Installing…',
+ installCancelled: 'Install cancelled: no sudo password was provided.',
+ installFailed: 'Install failed. Read the log above, or run the command on the host yourself.',
+ noPackageManager: 'No supported package manager (apt, dnf, pacman) was found on the gateway host.',
+ portalTitle: 'Screen',
+ portalOpen: 'Open',
+ heroStopped: 'Screen is off',
+ heroNotInstalled: 'Not installed on this host',
+ heroConnecting: 'Checking the screen…',
+ heroStale: 'Last seen — screen unreachable',
+ heroSuppressed: 'Hidden while someone has control',
+ heroOpenLive: 'Open live',
+ heroInstall: 'Install',
+ heroStart: 'Start',
+ portalWatching: 'Live · bot in control',
+ portalYouControl: 'Live · you are in control',
+ portalOtherControls: 'Live · another viewer in control',
+ portalStopped: 'Stopped',
+ portalNotInstalled: 'Not installed on host',
+ portalUnsupported: 'Not available on this host',
+ portalUnavailable: 'Update the bot\u2019s Hermes to use Screen',
+ unavailableTitle: 'Screen needs a newer Hermes',
+ stoppedTitle: 'Screen is off',
+ stoppedBody: 'Start this bot\u2019s desktop to watch what it does and take over when it needs you.',
+ start: 'Start screen',
+ attaching: 'Connecting to the screen\u2026',
+ streamLost: 'Screen stream ended',
+ reconnect: 'Reconnect',
+ takeOver: 'Take over',
+ handBack: 'Hand back',
+ handBackForce: 'Hand back (force)',
+ handBackForceHint: 'Release a lease held by a viewer that is no longer here, e.g. after a reload.',
+ openNeedsUpdate: 'Update Hermes Desktop to open bot screens.',
+ youControl: 'You are in control',
+ otherControls: 'Another viewer is in control',
+ agentControls: 'Bot is in control',
+ controlTaken: 'Another viewer took control. Watching only.',
+ handoffRequested: 'Bot needs you'
+ },
cron: {
filterHint:
'Scheduled jobs exist in this profile but none are tagged for this bot. Name a job "[bot:
] …" to show it here, or see them in Cron below.',
@@ -665,6 +763,54 @@ const ja: BotsMessages = {
searchHub: 'ハブを検索(コミュニティと既知のソース)…',
noMcpServers: '設定済みまたはカタログ内の MCP サーバーはありません。'
},
+ screen: {
+ title: '画面',
+ menu: '画面を開く',
+ unsupportedTitle: 'このホストにはボット画面がありません',
+ unsupportedBody: 'ボット画面は Linux のゲートウェイホストで動作します。このボットはホスト自身のディスプレイを使います。',
+ notInstalledTitle: '画面パッケージが不足しています',
+ notInstalledBody: 'このボットに画面を与えるには、ゲートウェイホストに TigerVNC と Xfce コアが必要です。ホストで実行:',
+ installHint: 'Hermes を実行しているユーザーとしてゲートウェイホスト上で実行されます。sudo は Hermes 経由で一度だけ求められます。',
+ install: 'ホストにインストール',
+ installing: 'インストール中…',
+ installCancelled: 'インストールを中止しました: sudo パスワードが入力されませんでした。',
+ installFailed: 'インストールに失敗しました。上のログを確認するか、ホストでコマンドを直接実行してください。',
+ noPackageManager: 'ゲートウェイホストに対応するパッケージマネージャー (apt, dnf, pacman) が見つかりません。',
+ portalTitle: 'スクリーン',
+ portalOpen: '開く',
+ heroStopped: '画面は停止中',
+ heroNotInstalled: 'このホストには未インストール',
+ heroConnecting: '画面を確認中…',
+ heroStale: '最終表示 — 画面に接続できません',
+ heroSuppressed: '他の人が操作中は非表示',
+ heroOpenLive: 'ライブで開く',
+ heroInstall: 'インストール',
+ heroStart: '開始',
+ portalWatching: 'ライブ · ボットが操作中',
+ portalYouControl: 'ライブ · あなたが操作中',
+ portalOtherControls: 'ライブ · 別のビューアーが操作中',
+ portalStopped: '停止中',
+ portalNotInstalled: 'ホストに未インストール',
+ portalUnsupported: 'このホストでは利用できません',
+ portalUnavailable: 'Screen を使うにはボットの Hermes を更新してください',
+ unavailableTitle: 'Screen には新しい Hermes が必要です',
+ stoppedTitle: '画面はオフです',
+ stoppedBody: 'このボットのデスクトップを起動すると、動作を見守り、必要なときに操作を引き継げます。',
+ start: '画面を起動',
+ attaching: '画面に接続中…',
+ streamLost: '画面ストリームが終了しました',
+ reconnect: '再接続',
+ takeOver: '引き継ぐ',
+ handBack: '戻す',
+ handBackForce: '強制的に戻す',
+ handBackForceHint: 'もう存在しないビューア(再読み込み後など)が保持しているリースを解放します。',
+ openNeedsUpdate: 'ボットの画面を開くには Hermes Desktop を更新してください。',
+ youControl: 'あなたが操作中',
+ otherControls: '別のビューアが操作中',
+ agentControls: 'ボットが操作中',
+ controlTaken: '別のビューアが操作を引き継ぎました。閲覧のみ。',
+ handoffRequested: 'ボットが助けを求めています'
+ },
cron: {
filterHint:
'このプロファイルには定期実行ジョブがありますが、このボット向けのタグが付いたものはありません。ジョブ名を「[bot:<名前>] …」にするとここに表示されます。下のCronでも確認できます。',
@@ -879,6 +1025,54 @@ const zh: BotsMessages = {
searchHub: '搜索技能中心(社区和常见来源)…',
noMcpServers: '未配置 MCP 服务器,目录中也没有。'
},
+ screen: {
+ title: '屏幕',
+ menu: '打开屏幕',
+ unsupportedTitle: '此主机没有机器人屏幕',
+ unsupportedBody: '机器人屏幕在 Linux 网关主机上运行。此机器人使用主机自身的显示器。',
+ notInstalledTitle: '缺少屏幕软件包',
+ notInstalledBody: '网关主机需要 TigerVNC 和 Xfce 核心组件才能为此机器人提供屏幕。在主机上运行:',
+ installHint: '在网关主机上以运行 Hermes 的用户身份执行;sudo 只会通过 Hermes 询问一次。',
+ install: '安装到主机',
+ installing: '正在安装…',
+ installCancelled: '安装已取消:未提供 sudo 密码。',
+ installFailed: '安装失败。请查看上方日志,或在主机上手动运行该命令。',
+ noPackageManager: '网关主机上未找到受支持的包管理器(apt、dnf、pacman)。',
+ portalTitle: '屏幕',
+ portalOpen: '打开',
+ heroStopped: '屏幕已关闭',
+ heroNotInstalled: '此主机未安装',
+ heroConnecting: '正在检查屏幕…',
+ heroStale: '最后画面 — 屏幕无法访问',
+ heroSuppressed: '有人控制时隐藏',
+ heroOpenLive: '实时打开',
+ heroInstall: '安装',
+ heroStart: '启动',
+ portalWatching: '直播 · 机器人控制中',
+ portalYouControl: '直播 · 你在控制',
+ portalOtherControls: '直播 · 其他查看者控制中',
+ portalStopped: '已停止',
+ portalNotInstalled: '主机未安装',
+ portalUnsupported: '此主机不可用',
+ portalUnavailable: '更新机器人的 Hermes 以使用屏幕',
+ unavailableTitle: '屏幕需要更新版的 Hermes',
+ stoppedTitle: '屏幕已关闭',
+ stoppedBody: '启动此机器人的桌面,观看它的操作,并在需要时接管。',
+ start: '启动屏幕',
+ attaching: '正在连接屏幕…',
+ streamLost: '屏幕流已结束',
+ reconnect: '重新连接',
+ takeOver: '接管',
+ handBack: '交还',
+ handBackForce: '强制交还',
+ handBackForceHint: '释放已不在场的查看者(例如重新加载后)持有的控制权。',
+ openNeedsUpdate: '更新 Hermes Desktop 以打开机器人屏幕。',
+ youControl: '你正在控制',
+ otherControls: '另一位查看者正在控制',
+ agentControls: '机器人正在控制',
+ controlTaken: '另一位查看者已接管控制。仅可观看。',
+ handoffRequested: '机器人需要你'
+ },
cron: {
filterHint:
'此配置档案中有定时任务,但没有一个标记给这个机器人。将任务命名为“[bot:<名称>] …”即可显示在这里,也可以在下方的 Cron 中查看。',
@@ -1093,6 +1287,54 @@ const zhHant: BotsMessages = {
searchHub: '搜尋技能中心(社群和常見來源)…',
noMcpServers: '未設定 MCP 伺服器,目錄中也沒有。'
},
+ screen: {
+ title: '螢幕',
+ menu: '開啟螢幕',
+ unsupportedTitle: '此主機沒有機器人螢幕',
+ unsupportedBody: '機器人螢幕在 Linux 閘道主機上執行。此機器人使用主機自身的顯示器。',
+ notInstalledTitle: '缺少螢幕套件',
+ notInstalledBody: '閘道主機需要 TigerVNC 與 Xfce 核心元件才能為此機器人提供螢幕。在主機上執行:',
+ installHint: '在閘道主機上以執行 Hermes 的使用者身分執行;sudo 只會透過 Hermes 詢問一次。',
+ install: '安裝到主機',
+ installing: '安裝中…',
+ installCancelled: '安裝已取消:未提供 sudo 密碼。',
+ installFailed: '安裝失敗。請查看上方日誌,或在主機上手動執行該指令。',
+ noPackageManager: '閘道主機上找不到受支援的套件管理器(apt、dnf、pacman)。',
+ portalTitle: '螢幕',
+ portalOpen: '開啟',
+ heroStopped: '螢幕已關閉',
+ heroNotInstalled: '此主機未安裝',
+ heroConnecting: '正在檢查螢幕…',
+ heroStale: '最後畫面 — 螢幕無法連線',
+ heroSuppressed: '有人控制時隱藏',
+ heroOpenLive: '即時開啟',
+ heroInstall: '安裝',
+ heroStart: '啟動',
+ portalWatching: '直播 · 機器人控制中',
+ portalYouControl: '直播 · 您在控制',
+ portalOtherControls: '直播 · 其他檢視者控制中',
+ portalStopped: '已停止',
+ portalNotInstalled: '主機未安裝',
+ portalUnsupported: '此主機不可用',
+ portalUnavailable: '更新機器人的 Hermes 以使用螢幕',
+ unavailableTitle: '螢幕需要較新版的 Hermes',
+ stoppedTitle: '螢幕已關閉',
+ stoppedBody: '啟動此機器人的桌面,觀看它的操作,並在需要時接手。',
+ start: '啟動螢幕',
+ attaching: '正在連線至螢幕…',
+ streamLost: '螢幕串流已結束',
+ reconnect: '重新連線',
+ takeOver: '接手',
+ handBack: '交還',
+ handBackForce: '強制交還',
+ handBackForceHint: '釋放已不在場的檢視者(例如重新載入後)持有的控制權。',
+ openNeedsUpdate: '更新 Hermes Desktop 以開啟機器人螢幕。',
+ youControl: '你正在控制',
+ otherControls: '另一位檢視者正在控制',
+ agentControls: '機器人正在控制',
+ controlTaken: '另一位檢視者已接手控制。僅可觀看。',
+ handoffRequested: '機器人需要你'
+ },
cron: {
filterHint:
'此設定檔中有排程工作,但沒有任何一個標記給這個機器人。將工作命名為「[bot:<名稱>] …」即可顯示在這裡,也可以在下方的 Cron 中查看。',
diff --git a/apps/desktop/src/plugins/hermes-bots/plugin.tsx b/apps/desktop/src/plugins/hermes-bots/plugin.tsx
index 8032e16134ffa..d472473b6dff9 100644
--- a/apps/desktop/src/plugins/hermes-bots/plugin.tsx
+++ b/apps/desktop/src/plugins/hermes-bots/plugin.tsx
@@ -15,8 +15,8 @@
* bot-initiated sends use `hermes -p chat --in ~ -c "Bot Chat"`.
*/
-import { CHAT_EMPTY_AREA, COMPOSER_AREAS, host, PALETTE_AREA, translateNow } from '@hermes/plugin-sdk'
-import type { ChatEmptyProps, PluginContext } from '@hermes/plugin-sdk'
+import { CHAT_EMPTY_AREA, COMPOSER_AREAS, host, PALETTE_AREA, SIDEBAR_PROFILE_GROUP_HEADER_AREA, translateNow } from '@hermes/plugin-sdk'
+import type { ChatEmptyProps, PluginContext, ProfileGroupRoute } from '@hermes/plugin-sdk'
import { startFaceClock, stopFaceClock } from './avatar'
import {
@@ -69,6 +69,7 @@ import {
sessionOwnsWorkspace
} from './roster-pane'
import { botRosterMeta, botWorkspaceOwnerKey, setBotsWorkspaceOwner } from './routing'
+import { ProfileGroupScreenPortal } from './screen-portal'
import { startHideSweepScheduler } from './session-sweep'
import { bumpBotOpenGeneration, getBotOpenGeneration, ID, setPluginCtx } from './shared'
import type { GroupChat, RosterRow } from './types'
@@ -363,6 +364,13 @@ export default {
// the meta/room storage hydrates above have landed; idempotent after that.
// (Feature-guarded: bare vm test harnesses have no setTimeout global.)
startHideSweepScheduler(ctx)
+ // Sessions sidebar: each gateway/profile group gets the profile's Screen portal
+ // above its sessions, so the bot's computer is reachable from either mode.
+ ctx.register({
+ id: 'screen-portal',
+ area: SIDEBAR_PROFILE_GROUP_HEADER_AREA,
+ data: { render: (route: ProfileGroupRoute) => }
+ })
ctx.register({
id: 'pane',
area: 'panes',
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts
new file mode 100644
index 0000000000000..f4f36759bf730
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts
@@ -0,0 +1,44 @@
+/**
+ * Two remote hosts can share the same `~/.hermes` path, so a `display.*` event
+ * matched on `profile_key` alone would let host B's take-over repaint host A's
+ * screen pane. The event must also have arrived on the bot's own connection.
+ */
+
+import { describe, expect, it, vi } from 'vitest'
+
+import type { RosterRow } from './types'
+
+const routeMock = vi.fn<() => { connectionId: string; profile: string } | null>(() => null)
+
+vi.mock('@hermes/plugin-sdk', () => ({
+ host: { requestProfile: vi.fn() },
+ resolveSiblingWsUrl: vi.fn()
+}))
+
+vi.mock('./routing', () => ({
+ botConnectionRoute: () => routeMock()
+}))
+
+import { isEventForBotScreen } from './screen-connection'
+
+const bot = { name: 'ops' } as RosterRow
+const key = '/home/hermes/.hermes'
+
+describe('isEventForBotScreen', () => {
+ it('ignores a same-profile-path event that arrived from another host', () => {
+ routeMock.mockReturnValue({ connectionId: 'conn-a', profile: 'ops' })
+
+ const fromB = { connectionId: 'conn-b', payload: { profile_key: key }, type: 'display.lease' }
+ const fromA = { connectionId: 'conn-a', payload: { profile_key: key }, type: 'display.lease' }
+
+ expect(isEventForBotScreen(bot, fromB, key)).toBe(false)
+ expect(isEventForBotScreen(bot, fromA, key)).toBe(true)
+ })
+
+ it('still matches the untagged local socket for a local bot', () => {
+ routeMock.mockReturnValue({ connectionId: 'local', profile: 'ops' })
+
+ expect(isEventForBotScreen(bot, { payload: { profile_key: key }, type: 'display.lease' }, key)).toBe(true)
+ expect(isEventForBotScreen(bot, { payload: { profile_key: '/other' }, type: 'display.lease' }, key)).toBe(false)
+ })
+})
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts
new file mode 100644
index 0000000000000..671fac03dd7e7
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts
@@ -0,0 +1,170 @@
+/**
+ * Bot Screen — connection plumbing for a bot's Bot Desktop (the headless Xfce
+ * screen its computer_use drives on the gateway host).
+ *
+ * Two legs share one authenticated origin: JSON-RPC (`display.*`) rides the
+ * bot's pooled gateway socket through `host.requestProfile`; the RFB stream
+ * rides a SIBLING WebSocket to `/api/display/ws`, minted per attach by
+ * `display.observe` (single-use, 30 s). noVNC's Websock takes ownership of the
+ * socket it is handed, so it can never share the JSON-RPC one — same reason
+ * voice playback opens `/api/audio/speak-stream` beside `/api/ws`.
+ */
+
+import { host, resolveSiblingWsUrl } from '@hermes/plugin-sdk'
+import type { PluginProfileRoute, RpcEvent } from '@hermes/plugin-sdk'
+
+import { botConnectionRoute } from './routing'
+import type { RosterRow } from './types'
+
+export interface DisplayLease {
+ holder: 'agent' | 'human'
+ /** Raw holder id — only older backends still broadcast it; newer ones send `viewer_hash`. */
+ viewer_id: null | string
+ /** First 12 hex of sha256(viewer_id): names the holder without leaking a usable id. */
+ viewer_hash?: null | string
+ since: number
+ reason: string
+ pending_handoff: null | string
+ /** Monotonic per transition; a lower epoch is an older snapshot, never newer truth. */
+ epoch?: number
+}
+
+export interface DisplayStatus {
+ profile: string
+ profile_key: string
+ supported: boolean
+ installed: boolean
+ missing: string[]
+ running: boolean
+ pid: null | number
+ display: null | string
+ socket: null | string
+ geometry: string
+ install_command: null | string
+ lease: DisplayLease
+}
+
+export interface DisplayThumbnail {
+ data_url: string | null
+ /** Set while a human holds the screen: the frame is withheld, not missing. */
+ suppressed?: 'human_has_control' | null
+}
+
+export interface DisplayObserveResult extends DisplayStatus {
+ ticket: string
+ path: string
+ /** Server-minted per attach: the only id the lease will ever be compared against. */
+ viewer_id: string
+}
+
+/** This window's identity for one attach: the minted id plus its lease-payload hash. */
+export interface ScreenViewer {
+ id: string
+ hash: string
+}
+
+const VIEWER_HASH_HEX = 12
+
+/** `viewer_hash` as the lease broadcasts it: first 12 hex of sha256(viewer_id). */
+export async function viewerHash(viewerId: string): Promise {
+ const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(viewerId))
+
+ return Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, '0'))
+ .join('')
+ .slice(0, VIEWER_HASH_HEX)
+}
+
+/** Does `viewer` (this window's attach) hold `lease`? Newer backends name the
+ * holder by hash only; a payload still carrying the raw id is compared raw. */
+export function leaseHeldBy(lease: DisplayLease | null | undefined, viewer: ScreenViewer | null | undefined): boolean {
+ if (!lease || !viewer || lease.holder !== 'human') {
+ return false
+ }
+
+ return lease.viewer_id != null ? lease.viewer_id === viewer.id : lease.viewer_hash === viewer.hash
+}
+
+/** JSON-RPC method-not-found: the bot's Hermes predates the `display.*` surface. */
+export function isDisplayUnavailable(error: unknown): boolean {
+ const record = typeof error === 'object' && error !== null ? (error as { code?: unknown; message?: unknown }) : null
+
+ if (record?.code === -32601) {
+ return true
+ }
+
+ const message = typeof record?.message === 'string' ? record.message.toLowerCase() : ''
+
+ return message.includes('method not found') || message.includes('method-not-found')
+}
+
+/** Bare-profile fallback so a v1 local bot (no registry route) still resolves. */
+export function botScreenRoute(bot: RosterRow): PluginProfileRoute | string {
+ return botConnectionRoute(bot) ?? bot.name
+}
+
+/**
+ * Does a `display.*` event belong to `bot`'s screen? Two hosts can share the same
+ * `~/.hermes` path, so the profile key alone is ambiguous: the event must also have
+ * arrived on the bot's registry connection (local/legacy events carry no tag).
+ */
+export function isEventForBotScreen(bot: RosterRow, event: RpcEvent, profileKey: null | string | undefined): boolean {
+ const payload = event.payload as { profile_key?: string } | undefined
+
+ if (!profileKey || payload?.profile_key !== profileKey) {
+ return false
+ }
+
+ const expected = botConnectionRoute(bot)?.connectionId ?? null
+ const actual = event.connectionId ?? null
+
+ return expected === actual || (expected === 'local' && actual === null)
+}
+
+export function displayRequest(bot: RosterRow, method: string, params: Record = {}): Promise {
+ return host.requestProfile(botScreenRoute(bot), method, params)
+}
+
+/**
+ * Hold the bot's pooled gateway socket open across a `display.*` sequence. The
+ * SDK disposes an inactive registry-routed socket once its request count hits
+ * zero, so without this the `display.install.*` / `display.lease` events that
+ * follow the request never arrive. Feature-detected: older hosts (and local
+ * routes, which never close) get a no-op release.
+ */
+export async function retainBotScreen(bot: RosterRow): Promise<() => void> {
+ const noop = () => undefined
+
+ if (typeof host.retainProfile !== 'function') {
+ return noop
+ }
+
+ try {
+ const release = await host.retainProfile(botScreenRoute(bot))
+
+ return typeof release === 'function' ? release : noop
+ } catch {
+ return noop
+ }
+}
+
+/**
+ * Resolve the RFB WebSocket URL for `bot`: the bot's gateway `/api/ws` origin
+ * (fresh credential for OAuth remotes) with the path swapped for the display
+ * bridge and the single-use display ticket attached.
+ */
+export async function resolveScreenWsUrl(bot: RosterRow, ticket: string): Promise {
+ const route = botConnectionRoute(bot)
+
+ // The /api/ws credential authenticated the RPC that minted the display ticket;
+ // the bridge authenticates on the ticket alone, so the gateway credential is
+ // dropped rather than spending a second one-shot ticket.
+ const url = new URL(
+ await resolveSiblingWsUrl({ connectionId: route?.connectionId ?? null, profile: route?.profile ?? bot.name }, '/api/display/ws', {
+ stripGatewayCredential: true
+ })
+ )
+
+ url.searchParams.set('display_ticket', ticket)
+
+ return url.toString()
+}
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx
new file mode 100644
index 0000000000000..647b866d4e553
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx
@@ -0,0 +1,167 @@
+/**
+ * Screen hero — the big preview of a bot's computer at the top of its pane.
+ *
+ * Shows a live thumbnail (one `display.thumbnail` JPEG every few seconds while
+ * the screen runs and the hero is on screen), or an honest placeholder for
+ * stopped / not installed / unsupported. Clicking it opens the full Screen pane
+ * where the user can take over.
+ */
+
+import { Codicon } from '@hermes/plugin-sdk'
+import { useEffect, useRef, useState } from 'react'
+
+import { botSelectionKey } from './data'
+import { useBots } from './i18n'
+import { displayRequest, type DisplayThumbnail } from './screen-connection'
+import { openBotScreen } from './screen-open'
+import { type PortalTone, useScreenPortalState } from './screen-portal'
+import type { BotMeta, RosterRow } from './types'
+
+const REFRESH_MS = 4000
+const STALE_AFTER = 3
+
+function useLiveThumbnail(bot: RosterRow, running: boolean) {
+ const [dataUrl, setDataUrl] = useState(null)
+ // Consecutive failed refreshes; past STALE_AFTER the frame is shown dimmed as "last seen" so a
+ // dead gateway never keeps looking live. Success resets it.
+ const [misses, setMisses] = useState(0)
+ // The backend withholds frames while a human holds the screen; that is a
+ // deliberate answer, not a failed refresh, so it never ages into "stale".
+ const [suppressed, setSuppressed] = useState(false)
+ const boxRef = useRef(null)
+
+ useEffect(() => {
+ if (!running) {
+ setDataUrl(null)
+ setMisses(0)
+ setSuppressed(false)
+
+ return
+ }
+
+ let cancelled = false
+ let timer: number | null = null
+ let visible = true
+
+ const observer =
+ typeof IntersectionObserver === 'undefined'
+ ? null
+ : new IntersectionObserver(entries => {
+ visible = entries.some(entry => entry.isIntersecting)
+ })
+
+ if (observer && boxRef.current) {
+ observer.observe(boxRef.current)
+ }
+
+ const tick = () => {
+ if (cancelled) {
+ return
+ }
+
+ if (!visible || document.hidden) {
+ timer = window.setTimeout(tick, REFRESH_MS)
+
+ return
+ }
+
+ void displayRequest(bot, 'display.thumbnail')
+ .then(result => {
+ if (!cancelled) {
+ setDataUrl(result.data_url)
+ setSuppressed(result.suppressed === 'human_has_control')
+ setMisses(0)
+ }
+ })
+ .catch(() => {
+ if (!cancelled) {
+ setMisses(prev => prev + 1) // the last good frame stays up, marked stale past STALE_AFTER
+ }
+ })
+ .finally(() => {
+ if (!cancelled) {
+ timer = window.setTimeout(tick, REFRESH_MS)
+ }
+ })
+ }
+
+ tick()
+
+ return () => {
+ cancelled = true
+ observer?.disconnect()
+
+ if (timer !== null) {
+ window.clearTimeout(timer)
+ }
+ }
+ }, [bot, running])
+
+ return { dataUrl, boxRef, stale: misses >= STALE_AFTER, suppressed }
+}
+
+const TONE_RING: Partial> = {
+ human: 'ring-2 ring-red-500/80',
+ other: 'ring-2 ring-amber-500/70'
+}
+
+export function ScreenHero({ bot, meta }: { bot: RosterRow; meta?: BotMeta | null }) {
+ // A profile switch must discard the previous owner's pixels before painting.
+ return
+}
+
+function ScreenHeroContent({ bot, meta }: { bot: RosterRow; meta?: BotMeta | null }) {
+ const t = useBots()
+ const { tone } = useScreenPortalState(bot)
+ const running = tone === 'live' || tone === 'human' || tone === 'other'
+ const { dataUrl, boxRef, stale, suppressed } = useLiveThumbnail(bot, running)
+
+ if (tone === 'unsupported' || tone === 'unavailable') {
+ return null
+ }
+
+ const caption = suppressed ? t.screen.heroSuppressed : stale ? t.screen.heroStale : {
+ live: t.screen.portalWatching,
+ human: t.screen.portalYouControl,
+ other: t.screen.portalOtherControls,
+ off: t.screen.heroStopped,
+ missing: t.screen.heroNotInstalled,
+ unsupported: t.screen.portalUnsupported,
+ unavailable: t.screen.portalUnavailable,
+ unknown: t.screen.heroConnecting
+ }[tone]
+
+ const cta = running ? t.screen.heroOpenLive : tone === 'missing' ? t.screen.heroInstall : tone === 'off' ? t.screen.heroStart : ''
+
+ return (
+
+ )
+}
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx
new file mode 100644
index 0000000000000..1f057dacce53e
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx
@@ -0,0 +1,107 @@
+/**
+ * An INACTIVE registry-routed bot's pooled socket is disposed by the SDK as soon
+ * as its request count hits zero — so `display.install.log/done` (and the
+ * pane's `display.lease` events) would never arrive. The install card must hold
+ * a retention from before `display.install` until the done event lands.
+ */
+
+import { act, fireEvent, render } from '@testing-library/react'
+import { beforeEach, expect, it, vi } from 'vitest'
+
+import type { DisplayStatus } from './screen-connection'
+import type { RosterRow } from './types'
+
+const calls = vi.hoisted(() => [] as string[])
+
+vi.mock('@hermes/plugin-sdk', async () => {
+ const { onGatewayEvent } = await import('../../contrib/events')
+
+ return {
+ Button: ({ children, ...props }: React.ButtonHTMLAttributes) => ,
+ Codicon: () => null,
+ GlyphSpinner: () => null,
+ resolveSiblingWsUrl: vi.fn(),
+ host: {
+ onEvent: onGatewayEvent,
+ requestProfile: vi.fn(async (_route: unknown, method: string) => {
+ calls.push(`request:${method}`)
+
+ return {}
+ }),
+ retainProfile: vi.fn(async () => {
+ calls.push('retain')
+
+ return () => {
+ calls.push('release')
+ }
+ })
+ }
+ }
+})
+vi.mock('./routing', () => ({
+ botConnectionRoute: () => ({ connectionId: 'host-a', profile: 'ops', targetProfile: 'ops' })
+}))
+vi.mock('./i18n', () => ({
+ useBots: () => ({
+ screen: {
+ notInstalledTitle: 'Missing',
+ notInstalledBody: 'Body',
+ installHint: 'Hint',
+ install: 'Install on host',
+ installing: 'Installing',
+ installCancelled: 'Cancelled',
+ installFailed: 'Failed',
+ noPackageManager: 'None'
+ }
+ })
+}))
+
+// eslint-disable-next-line no-restricted-imports
+import { emitGatewayEvent } from '../../contrib/events'
+
+import { ScreenInstallCard } from './screen-install'
+
+const bot: RosterRow = { name: 'ops', sourceScoped: true, connectionId: 'host-a', connectionKind: 'remote' }
+
+const status: DisplayStatus = {
+ profile: 'ops',
+ profile_key: '/home/hermes/.hermes',
+ supported: true,
+ installed: false,
+ missing: ['tigervnc'],
+ running: false,
+ pid: null,
+ display: null,
+ socket: null,
+ geometry: '1440x900',
+ install_command: 'sudo apt-get install -y tigervnc-standalone-server',
+ lease: { holder: 'agent', viewer_id: null, pending_handoff: null, since: 1, reason: '' }
+}
+
+beforeEach(() => {
+ calls.length = 0
+})
+
+it('retains the bot socket before display.install and releases it when the done event lands', async () => {
+ const onInstalled = vi.fn()
+ const view = render()
+
+ await act(async () => {
+ fireEvent.click(view.getByText('Install on host'))
+ })
+ expect(calls).toEqual(['retain', 'request:display.install'])
+
+ act(() =>
+ emitGatewayEvent({
+ type: 'display.install.done',
+ connectionId: 'host-a',
+ profile: 'ops',
+ payload: { profile_key: status.profile_key, code: 0, status: { ...status, installed: true } }
+ })
+ )
+ expect(calls).toEqual(['retain', 'request:display.install', 'release'])
+ expect(onInstalled).toHaveBeenCalledTimes(1)
+ view.unmount()
+ // Unmount after done must not double-release.
+ expect(calls.filter(call => call === 'release')).toHaveLength(1)
+})
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx
new file mode 100644
index 0000000000000..f5509527c5fb7
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx
@@ -0,0 +1,125 @@
+/**
+ * Bot Screen install card — installs the TigerVNC + Xfce packages on the bot's
+ * gateway host from inside Hermes Desktop.
+ *
+ * `display.install` starts the distro package command on the host; sudo, when
+ * needed, arrives as the same masked password card the terminal tool uses
+ * (`display.install.sudo.request`), so the password never touches this pane.
+ * Output streams back as `display.install.log`; `display.install.done` carries
+ * a fresh status the caller uses to flip the pane to "Start screen".
+ */
+
+import { Button, Codicon, GlyphSpinner, host } from '@hermes/plugin-sdk'
+import type { RpcEvent } from '@hermes/plugin-sdk'
+import { useCallback, useEffect, useRef, useState } from 'react'
+
+import { useBots } from './i18n'
+import { displayRequest, type DisplayStatus, isEventForBotScreen, retainBotScreen } from './screen-connection'
+import type { RosterRow } from './types'
+
+const LOG_KEEP = 200
+
+interface ScreenInstallCardProps {
+ bot: RosterRow
+ status: DisplayStatus
+ onInstalled: (status: DisplayStatus) => void
+}
+
+export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCardProps) {
+ const t = useBots()
+ const [phase, setPhase] = useState<'idle' | 'running' | 'failed'>('idle')
+ const [log, setLog] = useState([])
+ const [error, setError] = useState(null)
+ const logEnd = useRef(null)
+ // Keeps the bot's socket open from display.install until done/failed: the log
+ // and done events ride that socket, and the SDK closes an idle one otherwise.
+ const retention = useRef<(() => void) | null>(null)
+
+ const releaseRetention = useCallback(() => {
+ retention.current?.()
+ retention.current = null
+ }, [])
+
+ useEffect(() => releaseRetention, [releaseRetention])
+
+ useEffect(() => {
+ logEnd.current?.scrollIntoView({ block: 'end' })
+ }, [log])
+
+ useEffect(() => {
+ const offLog = host.onEvent('display.install.log', (event: RpcEvent) => {
+ const payload = event.payload as { line?: string } | undefined
+
+ if (isEventForBotScreen(bot, event, status.profile_key) && typeof payload?.line === 'string') {
+ const line = payload.line
+ setLog(prev => (prev.length >= LOG_KEEP ? [...prev.slice(1), line] : [...prev, line]))
+ }
+ })
+
+ const offDone = host.onEvent('display.install.done', (event: RpcEvent) => {
+ const payload = event.payload as { code?: number; status?: DisplayStatus } | undefined
+
+ if (!payload || !isEventForBotScreen(bot, event, status.profile_key)) {
+ return
+ }
+
+ releaseRetention()
+
+ if (payload.code === 0 && payload.status?.installed) {
+ setPhase('idle')
+ onInstalled(payload.status)
+ } else {
+ setPhase('failed')
+ setError(payload.code === -1 ? t.screen.installCancelled : t.screen.installFailed)
+ }
+ })
+
+ return () => {
+ offLog()
+ offDone()
+ }
+ }, [bot, onInstalled, releaseRetention, status.profile_key, t.screen.installCancelled, t.screen.installFailed])
+
+ const install = useCallback(async () => {
+ setPhase('running')
+ setLog([])
+ setError(null)
+
+ try {
+ retention.current = await retainBotScreen(bot)
+ await displayRequest(bot, 'display.install')
+ } catch (err) {
+ releaseRetention()
+ setPhase('failed')
+ setError(err instanceof Error ? err.message : String(err))
+ }
+ }, [bot, releaseRetention])
+
+ return (
+
+
+
{t.screen.notInstalledTitle}
+
{t.screen.notInstalledBody}
+ {status.install_command ? (
+
{status.install_command}
+ ) : (
+
{t.screen.noPackageManager}
+ )}
+ {status.install_command ? (
+
+ ) : null}
+ {log.length > 0 ? (
+
+ {log.join('\n')}
+
+
+ ) : null}
+ {error ?
{error}
: null}
+
{t.screen.installHint}
+
+
+ )
+}
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx
new file mode 100644
index 0000000000000..00b65c11e8f2f
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx
@@ -0,0 +1,214 @@
+import { act, fireEvent, render, renderHook } from '@testing-library/react'
+import { afterEach, beforeEach, expect, it, vi } from 'vitest'
+
+import type { RosterRow } from './types'
+
+vi.mock('@hermes/plugin-sdk', async () => {
+ const { useStore } = await import('@nanostores/react')
+ const { onGatewayEvent } = await import('../../contrib/events')
+
+ return {
+ Codicon: () => null,
+ useValue: useStore,
+ resolveSiblingWsUrl: vi.fn(),
+ host: { onEvent: vi.fn(onGatewayEvent), requestProfile: vi.fn() }
+ }
+})
+vi.mock('./data', async () => {
+ const { atom } = await import('nanostores')
+
+ return {
+ $lastRoster: atom([]),
+ botSelectionKey: (bot: RosterRow) =>
+ bot.sourceScoped || bot.remoteSource ? `${bot.connectionId}::${bot.name}` : bot.name
+ }
+})
+vi.mock('./i18n', () => ({
+ useBots: () => ({
+ screen: {
+ portalTitle: 'Screen',
+ portalWatching: 'Live',
+ portalYouControl: 'You control',
+ portalOtherControls: 'Other viewer',
+ heroOpenLive: 'Open live',
+ heroStale: 'Last seen',
+ heroSuppressed: 'Hidden while someone has control',
+ portalUnavailable: 'Update the bot',
+ heroConnecting: 'Connecting'
+ }
+ })
+}))
+vi.mock('./screen-open', () => ({ openBotScreen: vi.fn() }))
+
+import { host } from '@hermes/plugin-sdk'
+
+// Exercise the real event bus in this integration test, not a copied dispatcher.
+// eslint-disable-next-line no-restricted-imports
+import { emitGatewayEvent } from '../../contrib/events'
+
+import { $lastRoster } from './data'
+import type { DisplayStatus } from './screen-connection'
+import { ScreenHero } from './screen-hero'
+import { openBotScreen } from './screen-open'
+import { ProfileGroupScreenPortal, useScreenPortalState } from './screen-portal'
+import { $screenState, setScreenStatus } from './screen-state'
+
+const botA: RosterRow = { name: 'default', sourceScoped: true, connectionId: 'host-a', connectionKind: 'remote' }
+const botB: RosterRow = { ...botA, connectionId: 'host-b' }
+
+const status: DisplayStatus = {
+ profile: 'default',
+ profile_key: '/home/hermes/.hermes',
+ supported: true,
+ installed: true,
+ missing: [],
+ running: true,
+ pid: 42,
+ display: ':20',
+ socket: '/tmp/rfb.sock',
+ geometry: '1440x900',
+ install_command: null,
+ lease: { holder: 'agent', viewer_id: null, pending_handoff: null, since: 1, reason: '' }
+}
+
+beforeEach(() => {
+ $screenState.set({})
+ $lastRoster.set([])
+ vi.mocked(host.requestProfile).mockReset()
+ vi.mocked(openBotScreen).mockClear()
+ vi.spyOn(globalThis.document, 'hidden', 'get').mockReturnValue(false)
+})
+
+afterEach(() => {
+ vi.restoreAllMocks()
+ vi.useRealTimers()
+})
+
+it('applies lease events only from the owning host even when profile paths match', () => {
+ setScreenStatus(botA, status)
+ const view = renderHook(() => useScreenPortalState(botA))
+ const human = { ...status.lease, holder: 'human' as const, viewer_id: 'this-viewer' }
+
+ const emit = (connectionId: string, profileKey = status.profile_key) =>
+ act(() =>
+ emitGatewayEvent({
+ type: 'display.lease',
+ connectionId,
+ profile: 'default',
+ payload: { profile_key: profileKey, lease: human }
+ })
+ )
+
+ emit('host-b')
+ expect(view.result.current.lease?.holder).toBe('agent')
+ emit('host-a', '/another/profile')
+ expect(view.result.current.lease?.holder).toBe('agent')
+ emit('host-a')
+ expect(view.result.current.lease).toEqual(human)
+ view.unmount()
+})
+
+it('does not match a legacy profile group to a same-named remote bot', () => {
+ const legacy: RosterRow = { name: 'default' }
+ $lastRoster.set([botB, legacy])
+ setScreenStatus(botB, status)
+ setScreenStatus(legacy, status)
+ const view = render()
+
+ fireEvent.click(view.getByRole('button'))
+ expect(openBotScreen).toHaveBeenCalledWith(legacy, null)
+ view.rerender()
+ fireEvent.click(view.getByRole('button'))
+ expect(openBotScreen).toHaveBeenLastCalledWith(botB, null)
+ view.unmount()
+})
+
+it.each(['pending', 'rejected'])('never displays host A pixels under host B while B is %s', async state => {
+ setScreenStatus(botA, status)
+ setScreenStatus(botB, status)
+ vi.mocked(host.requestProfile)
+ .mockResolvedValueOnce({ data_url: 'data:image/jpeg;base64,HOST_A' })
+ .mockImplementationOnce(() => (state === 'pending' ? new Promise(() => {}) : Promise.reject(new Error('offline'))))
+ const view = render()
+ await act(async () => {})
+ expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_A')
+
+ view.rerender()
+ await act(async () => {})
+ expect(view.container.querySelector('img')).toBeNull()
+ view.unmount()
+})
+
+it('discards a late thumbnail from the previous owner and retains a same-owner frame on refresh failure', async () => {
+ vi.useFakeTimers()
+ setScreenStatus(botA, status)
+ setScreenStatus(botB, status)
+ let finishA!: (value: unknown) => void
+ vi.mocked(host.requestProfile)
+ .mockImplementationOnce(
+ () =>
+ new Promise(resolve => {
+ finishA = resolve
+ })
+ )
+ .mockResolvedValueOnce({ data_url: 'data:image/jpeg;base64,HOST_B' })
+ .mockRejectedValue(new Error('offline'))
+ const view = render()
+ view.rerender()
+ await act(async () => {})
+ await act(async () => {
+ finishA({ data_url: 'data:image/jpeg;base64,HOST_A' })
+ })
+ expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_B')
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(12_000)
+ })
+ expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_B')
+ expect(view.getByRole('button').getAttribute('aria-label')).toContain('Last seen')
+ view.unmount()
+})
+
+it('captions a suppressed thumbnail as hidden-while-controlled and never ages it into stale', async () => {
+ vi.useFakeTimers()
+ setScreenStatus(botA, status)
+ vi.mocked(host.requestProfile).mockResolvedValue({ data_url: null, suppressed: 'human_has_control' })
+ const view = render()
+ await act(async () => {})
+ expect(view.getByRole('button').getAttribute('aria-label')).toContain('Hidden while someone has control')
+
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(20_000)
+ })
+ expect(view.getByRole('button').getAttribute('aria-label')).toContain('Hidden while someone has control')
+ expect(view.getByRole('button').getAttribute('aria-label')).not.toContain('Last seen')
+ view.unmount()
+})
+
+it('settles on an older backend without display.*: portal tone is unavailable and the hero renders nothing', async () => {
+ vi.mocked(host.requestProfile).mockRejectedValue(Object.assign(new Error('Method not found: display.status'), { code: -32601 }))
+ const hook = renderHook(() => useScreenPortalState(botA))
+ await act(async () => {})
+ expect(hook.result.current.tone).toBe('unavailable')
+ hook.rerender()
+ await act(async () => {})
+ expect(vi.mocked(host.requestProfile)).toHaveBeenCalledTimes(1)
+ hook.unmount()
+
+ const view = render()
+ await act(async () => {})
+ expect(view.container.firstChild).toBeNull()
+ view.unmount()
+})
+
+it('a sidebar group portal keeps its lease subscription across parent re-renders (no per-paint row rebuild)', async () => {
+ vi.mocked(host.requestProfile).mockResolvedValue(status)
+ const view = render()
+ await act(async () => {})
+ const subscriptions = vi.mocked(host.onEvent).mock.calls.length
+
+ view.rerender()
+ view.rerender()
+ await act(async () => {})
+ expect(vi.mocked(host.onEvent).mock.calls.length).toBe(subscriptions)
+ view.unmount()
+})
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-open.tsx b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx
new file mode 100644
index 0000000000000..9d05539c4d889
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx
@@ -0,0 +1,46 @@
+/**
+ * Open a bot's Screen as a main-window workspace tab (host.openWorkspace),
+ * one tab per bot; a second open refocuses the existing tab.
+ */
+
+import { host } from '@hermes/plugin-sdk'
+
+import { botSelectionKey } from './data'
+import { botsText } from './i18n'
+import { displayName } from './labels'
+import { BotScreenPane } from './screen-pane'
+import { ID } from './shared'
+import type { BotMeta, RosterRow } from './types'
+
+const openTabs = new Map void>()
+
+export function screenPaneId(bot: RosterRow): string {
+ return `plugin-workspace:${ID}:screen:${botSelectionKey(bot)}`
+}
+
+export function openBotScreen(bot: RosterRow, meta?: BotMeta | null): void {
+ if (typeof host.openWorkspace !== 'function') {
+ host.notify({ kind: 'info', message: botsText().screen.openNeedsUpdate })
+
+ return
+ }
+
+ const key = botSelectionKey(bot)
+
+ if (openTabs.has(key)) {
+ host.revealPane(screenPaneId(bot))
+
+ return
+ }
+
+ const close = host.openWorkspace(`${ID}:screen:${key}`, {
+ title: `${displayName(bot, meta ?? null)} · ${botsText().screen.title}`,
+ minWidth: '28rem',
+ render: () => ,
+ onClose: () => {
+ openTabs.delete(key)
+ }
+ })
+
+ openTabs.set(key, close)
+}
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx
new file mode 100644
index 0000000000000..95eaa59522981
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx
@@ -0,0 +1,201 @@
+import { act, fireEvent, render, waitFor } from '@testing-library/react'
+import { afterEach, beforeEach, expect, it, vi } from 'vitest'
+
+import type { DisplayStatus } from './screen-connection'
+import type * as ScreenConnection from './screen-connection'
+import type { RosterRow } from './types'
+
+const sockets = vi.hoisted(
+ () => [] as Array<{ closeCodes: number[]; closed: boolean; close: (code?: number) => void; serverClose: (code: number) => void }>
+)
+
+const rfbs = vi.hoisted(() => [] as Array<{ emit: (type: string, detail?: unknown) => void }>)
+const retention = vi.hoisted(() => ({ held: 0 }))
+
+vi.mock('@hermes/plugin-sdk', async () => {
+ const { useStore } = await import('@nanostores/react')
+ const { onGatewayEvent } = await import('../../contrib/events')
+
+ return {
+ Button: ({ children, ...props }: React.ButtonHTMLAttributes) => (
+
+ ),
+ Codicon: () => null,
+ GlyphSpinner: () => null,
+ EmptyState: () => null,
+ useValue: useStore,
+ host: {
+ onEvent: onGatewayEvent,
+ retainProfile: async () => {
+ retention.held += 1
+
+ return () => {
+ retention.held -= 1
+ }
+ }
+ }
+ }
+})
+vi.mock('./routing', () => ({
+ botConnectionRoute: () => ({ connectionId: 'host-a', profile: 'default', targetProfile: 'default' })
+}))
+vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name }))
+vi.mock('./i18n', () => ({
+ useBots: () => ({
+ screen: {
+ title: 'Screen',
+ controlTaken: 'Another viewer took control',
+ youControl: 'You control',
+ handBack: 'Hand back',
+ takeOver: 'Take over',
+ reconnect: 'Reconnect',
+ streamLost: 'Stream lost'
+ }
+ })
+}))
+vi.mock('./screen-connection', async importActual => ({
+ // Real pure helpers (viewerHash / leaseHeldBy); only the gateway legs are faked.
+ ...(await importActual()),
+ displayRequest: vi.fn(),
+ resolveScreenWsUrl: vi.fn(async () => 'ws://localhost/api/display/ws'),
+ isEventForBotScreen: () => false
+}))
+vi.mock('@novnc/novnc', () => ({
+ default: class {
+ private listeners = new Map void>>()
+ constructor(
+ _target: HTMLElement,
+ private socket: { close: () => void }
+ ) {
+ rfbs.push(this)
+ }
+ emit(type: string, detail?: unknown) {
+ for (const listener of this.listeners.get(type) ?? []) {
+ listener({ detail })
+ }
+ }
+ addEventListener(type: string, callback: (event: { detail?: unknown }) => void) {
+ this.listeners.set(type, [...(this.listeners.get(type) ?? []), callback])
+
+ if (type === 'connect') {
+ queueMicrotask(() => callback({}))
+ }
+ }
+ // noVNC 1.7 disconnects its WebSocket without a close code.
+ disconnect() {
+ this.socket.close()
+ }
+ focus() {}
+ }
+}))
+
+import { displayRequest } from './screen-connection'
+import { BotScreenPane } from './screen-pane'
+import { $screenState } from './screen-state'
+
+const bot: RosterRow = { name: 'default' }
+
+const status: DisplayStatus = {
+ profile: 'default',
+ profile_key: '/home/hermes/.hermes',
+ supported: true,
+ installed: true,
+ missing: [],
+ running: true,
+ pid: 42,
+ display: ':20',
+ socket: '/tmp/rfb.sock',
+ geometry: '1440x900',
+ install_command: null,
+ lease: { holder: 'human', viewer_id: 'this-viewer', pending_handoff: null, since: 1, reason: '' }
+}
+
+beforeEach(() => {
+ $screenState.set({})
+ sockets.length = 0
+ rfbs.length = 0
+ retention.held = 0
+ vi.mocked(displayRequest)
+ .mockReset()
+ .mockResolvedValue({ ...status, ticket: 'test-ticket', viewer_id: 'this-viewer' })
+ vi.stubGlobal(
+ 'WebSocket',
+ class {
+ closeCodes: number[] = []
+ closed = false
+ private onClose: Array<(event: { code: number }) => void> = []
+ constructor() {
+ sockets.push(this)
+ }
+ addEventListener(type: string, listener: (event: { code: number }) => void) {
+ if (type === 'close') {
+ this.onClose.push(listener)
+ }
+ }
+ // The bridge closing us: the raw close frame reaches our listener, then noVNC
+ // reports a statusless `disconnect` — the code is only on the socket event.
+ serverClose(code: number) {
+ this.closed = true
+
+ for (const listener of this.onClose) {
+ listener({ code })
+ }
+ }
+ close(code?: number) {
+ // Subsequent close calls cannot replace the frame already sent to the server.
+ if (this.closed) {
+ return
+ }
+
+ this.closed = true
+ this.closeCodes.push(code ?? 1005)
+ }
+ }
+ )
+})
+
+afterEach(() => vi.unstubAllGlobals())
+
+it('sends an intentional close before noVNC can send its statusless close on pane unmount', async () => {
+ const view = render()
+ await waitFor(() => expect(sockets).toHaveLength(1))
+ await act(async () => {})
+ view.unmount()
+ expect(sockets[0].closeCodes).toEqual([1000])
+})
+
+it('pins the bot socket for the attach lifetime and lets go on unmount', async () => {
+ const view = render()
+ await waitFor(() => expect(sockets).toHaveLength(1))
+ await act(async () => {})
+ expect(retention.held).toBe(1)
+ fireEvent.click(view.getByTitle('Reconnect'))
+ await waitFor(() => expect(sockets).toHaveLength(2))
+ expect(retention.held).toBe(1)
+ view.unmount()
+ expect(retention.held).toBe(0)
+})
+
+it('does not hand back while replacing a stream to reconnect the same viewer', async () => {
+ const view = render()
+ await waitFor(() => expect(sockets).toHaveLength(1))
+ await act(async () => {})
+ fireEvent.click(view.getByTitle('Reconnect'))
+ await waitFor(() => expect(sockets).toHaveLength(2))
+ expect(sockets[0].closeCodes).toEqual([1005])
+ expect(sockets[1].closed).toBe(false)
+ view.unmount()
+})
+
+it('shows the control-taken overlay from the bridge close code, which noVNC does not forward', async () => {
+ const view = render()
+ await waitFor(() => expect(sockets).toHaveLength(1))
+ await act(async () => {})
+
+ act(() => {
+ sockets[0].serverClose(4000)
+ rfbs[0].emit('disconnect', { clean: true })
+ })
+ expect(view.getByText('Another viewer took control')).toBeTruthy()
+ view.unmount()
+})
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx
new file mode 100644
index 0000000000000..1dea30e2a4e58
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx
@@ -0,0 +1,364 @@
+/**
+ * Bot Screen pane — live view of a bot's headless desktop with Take over / Hand back.
+ *
+ * State authority: the BACKEND owns runtime + lease (`display.status`, pushed
+ * as `display.lease` events); this pane paints a cache of it. The RFB stream
+ * is a sibling WebSocket handed to noVNC's RFB; `viewOnly` here is UX only —
+ * the gateway drops input from anyone but the lease holder.
+ *
+ * Every attach spends a single-use ticket, so a lease flip that the bridge
+ * answers with close 4000 (`control-taken`) simply re-attaches in watch mode.
+ */
+
+import { Button, Codicon, EmptyState, GlyphSpinner, host, useValue } from '@hermes/plugin-sdk'
+import type { RpcEvent } from '@hermes/plugin-sdk'
+import { useCallback, useEffect, useRef, useState } from 'react'
+
+import { useBots } from './i18n'
+import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isDisplayUnavailable, isEventForBotScreen, leaseHeldBy, resolveScreenWsUrl, retainBotScreen, viewerHash } from './screen-connection'
+import { ScreenInstallCard } from './screen-install'
+import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenUnavailable, setScreenViewer } from './screen-state'
+import type { RosterRow } from './types'
+
+type RfbLike = {
+ viewOnly: boolean
+ scaleViewport: boolean
+ resizeSession: boolean
+ focusOnClick: boolean
+ background: string
+ qualityLevel: number
+ addEventListener: (type: string, handler: (event: { detail?: { clean?: boolean; reason?: string } }) => void) => void
+ disconnect: () => void
+ focus: () => void
+}
+
+type ConnState = 'idle' | 'attaching' | 'live' | 'control-taken' | 'error'
+
+/** Bridge close code when another viewer took the lease (mirrors tui_gateway display bridge). */
+const CLOSE_CONTROL_TAKEN = 4000
+
+async function loadRfb(): Promise) => RfbLike> {
+ const mod = (await import('@novnc/novnc')) as unknown as { default: new (...args: never[]) => RfbLike }
+
+ return mod.default as unknown as new (target: HTMLElement, socket: WebSocket, options?: Record) => RfbLike
+}
+
+export function BotScreenPane({ bot }: { bot: RosterRow }) {
+ const t = useBots()
+ const screen = useValue($screenState)
+ const state = screenStateFor(screen, bot)
+ const status = state?.status ?? null
+ const lease = state?.lease ?? status?.lease ?? null
+ // The server mints this window's viewer id per attach (`display.observe`); the lease
+ // names its holder by hash, so a reload can never inherit a stale holder's authority.
+ const viewer = state?.viewer ?? null
+ const iHold = leaseHeldBy(lease, viewer)
+
+ const canvasHost = useRef(null)
+ const rfb = useRef(null)
+ const socket = useRef(null)
+ // Pins the bot's pooled gateway socket for the attach lifetime so display.lease
+ // events keep arriving for an inactive registry-routed bot.
+ const retention = useRef<(() => void) | null>(null)
+ const [conn, setConn] = useState('idle')
+ const [error, setError] = useState(null)
+ const [busy, setBusy] = useState(false)
+ const attachGeneration = useRef(0)
+
+ const refresh = useCallback(async () => {
+ try {
+ const next = await displayRequest(bot, 'display.status')
+ setScreenStatus(bot, next)
+ setError(null)
+ } catch (err) {
+ if (isDisplayUnavailable(err)) {
+ setScreenUnavailable(bot)
+ }
+
+ setError(err instanceof Error ? err.message : String(err))
+ }
+ }, [bot])
+
+ useEffect(() => {
+ void refresh()
+
+ return host.onEvent('display.lease', (event: RpcEvent) => {
+ const payload = event.payload as { lease?: DisplayLease } | undefined
+
+ if (payload?.lease && isEventForBotScreen(bot, event, status?.profile_key)) {
+ setScreenLease(bot, payload.lease)
+ }
+ })
+ }, [bot, refresh, status?.profile_key])
+
+ const detach = useCallback((handBack = false) => {
+ attachGeneration.current += 1
+
+ // noVNC closes without a status. Intentional pane closure must send 1000
+ // first; reconnect teardown must keep the human lease instead.
+ if (handBack) {
+ socket.current?.close(1000)
+ }
+
+ rfb.current?.disconnect()
+ rfb.current = null
+ socket.current?.close()
+ socket.current = null
+ retention.current?.()
+ retention.current = null
+ }, [])
+
+ const attach = useCallback(async () => {
+ if (!canvasHost.current) {
+ return
+ }
+
+ detach()
+ const generation = attachGeneration.current
+ setConn('attaching')
+ setError(null)
+
+ try {
+ // Load the client BEFORE dialing: noVNC's Websock installs its own `onopen`, so a socket that
+ // opened while the dynamic import was still in flight never hands it the open event.
+ const Rfb = await loadRfb()
+ const retain = await retainBotScreen(bot)
+
+ if (generation !== attachGeneration.current) {
+ retain()
+
+ return
+ }
+
+ retention.current = retain
+ const observe = await displayRequest(bot, 'display.observe')
+ const minted = { id: observe.viewer_id, hash: await viewerHash(observe.viewer_id) }
+ setScreenStatus(bot, observe)
+ const url = await resolveScreenWsUrl(bot, observe.ticket)
+
+ if (generation !== attachGeneration.current || !canvasHost.current) {
+ return
+ }
+
+ setScreenViewer(bot, minted)
+ const ws = new WebSocket(url)
+ ws.binaryType = 'arraybuffer'
+ socket.current = ws
+ // noVNC 1.7's `disconnect` detail carries only {clean}; the bridge's verdict lives in
+ // the raw close frame (4000 = control-taken). Listen here, before RFB installs its
+ // own `onclose`, so the code is known by the time the disconnect event fires.
+ let closeCode = 0
+ ws.addEventListener('close', event => {
+ closeCode = event.code
+ })
+ const client = new Rfb(canvasHost.current, ws, { shared: true })
+ client.scaleViewport = true
+ client.resizeSession = false
+ client.focusOnClick = true
+ client.background = 'transparent'
+ client.qualityLevel = 7
+ client.viewOnly = !leaseHeldBy(observe.lease, minted)
+ client.addEventListener('connect', () => {
+ if (generation === attachGeneration.current) {
+ setConn('live')
+ }
+ })
+ client.addEventListener('disconnect', event => {
+ // noVNC logs "Tried changing state of a disconnected RFB object" if we later call
+ // disconnect() on a client that already closed itself (eviction, stream loss).
+ if (rfb.current === client) {
+ rfb.current = null
+ }
+
+ if (generation !== attachGeneration.current) {
+ return
+ }
+
+ const reason = event.detail?.reason ?? ''
+
+ if (closeCode === CLOSE_CONTROL_TAKEN || reason.includes('control-taken')) {
+ setConn('control-taken')
+ } else if (event.detail?.clean) {
+ setConn('idle')
+ } else {
+ setConn('error')
+ setError(reason || t.screen.streamLost)
+ }
+
+ void refresh()
+ })
+ rfb.current = client
+ } catch (err) {
+ if (generation === attachGeneration.current) {
+ setConn('error')
+ setError(err instanceof Error ? err.message : String(err))
+ }
+ }
+ }, [bot, detach, refresh, t.screen.streamLost])
+
+ // Visibility is not lifecycle: the stream stays attached while the pane is
+ // hidden; only unmount tears it down (and hands control back server-side).
+ useEffect(() => () => detach(true), [detach])
+
+ useEffect(() => {
+ if (status?.running && conn === 'idle') {
+ void attach()
+ }
+ }, [attach, conn, status?.running])
+
+ useEffect(() => {
+ if (rfb.current) {
+ rfb.current.viewOnly = !iHold
+
+ if (iHold) {
+ rfb.current.focus()
+ }
+ }
+ }, [iHold])
+
+ const start = useCallback(async () => {
+ setBusy(true)
+
+ try {
+ const next = await displayRequest(bot, 'display.start')
+ setScreenStatus(bot, next)
+ setConn('idle')
+ } catch (err) {
+ setError(err instanceof Error ? err.message : String(err))
+ } finally {
+ setBusy(false)
+ }
+ }, [bot])
+
+ const takeOver = useCallback(async () => {
+ setBusy(true)
+
+ try {
+ const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.acquire', { viewer_id: viewer?.id })
+ setScreenLease(bot, result.lease)
+
+ if (conn !== 'live') {
+ void attach()
+ }
+ } catch (err) {
+ setError(err instanceof Error ? err.message : String(err))
+ } finally {
+ setBusy(false)
+ }
+ }, [attach, bot, conn, viewer?.id])
+
+ // `force` is the escape hatch for a lease this window no longer owns (a reload
+ // minted a fresh viewer id; the old one still holds): the server refuses a
+ // plain release from anyone but the holder.
+ const handBack = useCallback(
+ async (force = false) => {
+ setBusy(true)
+
+ try {
+ const params = force ? { force: true } : { viewer_id: viewer?.id }
+ const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', params)
+ setScreenLease(bot, result.lease)
+ } catch (err) {
+ setError(err instanceof Error ? err.message : String(err))
+ } finally {
+ setBusy(false)
+ }
+ },
+ [bot, viewer?.id]
+ )
+
+ if (state?.unavailable) {
+ return
+ }
+
+ if (status && !status.supported) {
+ return
+ }
+
+ if (status && !status.installed) {
+ return setScreenStatus(bot, next)} status={status} />
+ }
+
+ if (status && !status.running) {
+ return (
+
+
+
{t.screen.stoppedTitle}
+
{t.screen.stoppedBody}
+
+ {error ?
{error}
: null}
+
+
+ )
+ }
+
+ const humanOther = lease?.holder === 'human' && !iHold
+
+ return (
+
+
+
+ {t.screen.title}
+ {status?.display ? {status.display} · {status.geometry} : null}
+
+ {lease?.pending_handoff ? (
+
+ {t.screen.handoffRequested}
+
+ ) : lease?.holder === 'human' && lease.reason ? (
+ // The agent's ask stays readable WHILE the human acts, not only before Take over.
+
+ {lease.reason}
+
+ ) : null}
+ {iHold ? (
+ {t.screen.youControl}
+ ) : humanOther ? (
+ {t.screen.otherControls}
+ ) : (
+ {t.screen.agentControls}
+ )}
+ {iHold ? (
+
+ ) : (
+ <>
+ {humanOther ? (
+
+ ) : null}
+
+ >
+ )}
+
+
+
+ {/* data-terminal: the same keyboard-ownership marker the terminal pane uses, so the app's
+ type-to-focus / bare-key shortcuts never steal keystrokes meant for the remote screen.
+ data-remote-screen: tells the ⌘W close-tab router this is NOT a local terminal tab —
+ the chord belongs to the remote desktop, nothing local should close. */}
+
+ {conn === 'attaching' ? (
+
+ {t.screen.attaching}
+
+ ) : null}
+ {conn === 'control-taken' ? (
+
{t.screen.controlTaken}
+ ) : null}
+ {conn === 'error' && error ? (
+
{error}
+ ) : null}
+
+
+ )
+}
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx
new file mode 100644
index 0000000000000..7afa011f24364
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx
@@ -0,0 +1,191 @@
+/**
+ * Screen portal — the compact "this bot's computer" box that sits above a
+ * bot's routines and on a gateway/profile group in the Sessions sidebar.
+ * One click opens the live Screen pane; the subtitle says whether the screen
+ * is live and who holds it, so the user knows before opening whether they
+ * are about to watch, take over, install or start.
+ *
+ * Reads the same per-bot cache the pane paints (`$screenState`) and refreshes
+ * it once on mount so a bot the user never opened still shows real state.
+ */
+
+import { Codicon, host, useValue } from '@hermes/plugin-sdk'
+import type { RpcEvent } from '@hermes/plugin-sdk'
+import type { ProfileGroupRoute } from '@hermes/plugin-sdk'
+import { useEffect, useMemo } from 'react'
+
+import { $lastRoster } from './data'
+import { useBots } from './i18n'
+import { resolveBotConnectionRoute } from './routing'
+import { type DisplayLease, displayRequest, type DisplayStatus, isDisplayUnavailable, isEventForBotScreen, leaseHeldBy, type ScreenViewer } from './screen-connection'
+import { openBotScreen } from './screen-open'
+import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenUnavailable } from './screen-state'
+import type { BotMeta, RosterRow } from './types'
+
+export type PortalTone = 'live' | 'human' | 'other' | 'off' | 'missing' | 'unsupported' | 'unavailable' | 'unknown'
+
+/** Pure: map cached status + lease (+ this window's minted viewer, if attached) to what the portal says. */
+export function portalTone(status: DisplayStatus | null, lease: DisplayLease | null, viewer: ScreenViewer | null = null, unavailable = false): PortalTone {
+ if (unavailable) {
+ return 'unavailable'
+ }
+
+ if (!status) {
+ return 'unknown'
+ }
+
+ if (!status.supported) {
+ return 'unsupported'
+ }
+
+ if (!status.installed) {
+ return 'missing'
+ }
+
+ if (!status.running) {
+ return 'off'
+ }
+
+ if (lease?.holder === 'human') {
+ return leaseHeldBy(lease, viewer) ? 'human' : 'other'
+ }
+
+ return 'live'
+}
+
+const TONE_ICON: Record = {
+ live: 'device-desktop',
+ human: 'record-keys',
+ other: 'eye',
+ off: 'debug-stop',
+ missing: 'cloud-download',
+ unsupported: 'circle-slash',
+ unavailable: 'circle-slash',
+ unknown: 'device-desktop'
+}
+
+const TONE_DOT: Record = {
+ live: 'bg-emerald-500',
+ human: 'bg-red-500',
+ other: 'bg-amber-500',
+ off: 'bg-(--ui-text-quaternary)',
+ missing: 'bg-(--ui-text-quaternary)',
+ unsupported: 'bg-(--ui-text-quaternary)',
+ unavailable: 'bg-(--ui-text-quaternary)',
+ unknown: 'bg-(--ui-text-quaternary)'
+}
+
+export function useScreenPortalState(bot: RosterRow) {
+ const all = useValue($screenState)
+ const state = screenStateFor(all, bot)
+ const status = state?.status ?? null
+ const profileKey = status?.profile_key
+
+ useEffect(() => {
+ if (status || state?.unavailable) {
+ return
+ }
+
+ let cancelled = false
+
+ void displayRequest(bot, 'display.status')
+ .then(next => {
+ if (!cancelled) {
+ setScreenStatus(bot, next)
+ }
+ })
+ .catch((error: unknown) => {
+ // An older Hermes without display.* is a settled answer (hide the surface);
+ // an offline bot is transient and stays in its unknown state.
+ if (!cancelled && isDisplayUnavailable(error)) {
+ setScreenUnavailable(bot)
+ }
+ })
+
+ return () => {
+ cancelled = true
+ }
+ }, [bot, state?.unavailable, status])
+
+ useEffect(
+ () =>
+ host.onEvent('display.lease', (event: RpcEvent) => {
+ const payload = event.payload as { profile_key?: string; lease?: DisplayLease } | undefined
+
+ if (payload?.lease && isEventForBotScreen(bot, event, profileKey)) {
+ setScreenLease(bot, payload.lease)
+ }
+ }),
+ [bot, profileKey]
+ )
+
+ return { status, lease: state?.lease ?? null, tone: portalTone(status, state?.lease ?? null, state?.viewer ?? null, state?.unavailable) }
+}
+
+export function ScreenPortal({ bot, meta, compact = false }: { bot: RosterRow; meta?: BotMeta | null; compact?: boolean }) {
+ const t = useBots()
+ const { status, tone } = useScreenPortalState(bot)
+
+ const subtitle = {
+ live: t.screen.portalWatching,
+ human: t.screen.portalYouControl,
+ other: t.screen.portalOtherControls,
+ off: t.screen.portalStopped,
+ missing: t.screen.portalNotInstalled,
+ unsupported: t.screen.portalUnsupported,
+ unavailable: t.screen.portalUnavailable,
+ unknown: status?.display ?? ''
+ }[tone]
+
+ if ((tone === 'unsupported' || tone === 'unavailable') && compact) {
+ return null
+ }
+
+ return (
+
+ )
+}
+
+/** Sessions-sidebar variant: the gateway/profile group hands us its route; find the
+ * bot that owns it, or synthesize a scoped row so a profile outside the current
+ * roster filter still gets its portal (the portal only needs a routable row). */
+export function ProfileGroupScreenPortal({ route }: { route: ProfileGroupRoute }) {
+ const roster = useValue($lastRoster)
+ const { connectionId, profile } = route
+
+ // Stable row identity: the portal's effects key on `bot`, so a synthesized row
+ // rebuilt every render would re-subscribe the lease listener on every sidebar paint.
+ const bot = useMemo(
+ () =>
+ roster.find(row => {
+ const resolved = resolveBotConnectionRoute(row)
+
+ return resolved.route
+ ? resolved.route.profile === profile && resolved.route.connectionId === (connectionId ?? 'local')
+ : row.name === profile && connectionId === null
+ }) ??
+ (connectionId
+ ? ({ name: profile, sourceScoped: true, connectionId, connectionKind: connectionId === 'local' ? 'local' : 'remote' } as RosterRow)
+ : ({ name: profile } as RosterRow)),
+ [connectionId, profile, roster]
+ )
+
+ return
+}
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts
new file mode 100644
index 0000000000000..6e294921852ff
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts
@@ -0,0 +1,61 @@
+/**
+ * Lease ordering: `display.lease` events and `display.status` replies race on
+ * the wire. A slower status reply describing an OLDER lease must never roll
+ * back the newer event — the backend's monotonic `epoch` is the tiebreak.
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import type { DisplayLease, DisplayStatus } from './screen-connection'
+import type { RosterRow } from './types'
+
+vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name }))
+
+import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state'
+
+const bot: RosterRow = { name: 'ops' }
+
+const agent: DisplayLease = { holder: 'agent', viewer_id: null, viewer_hash: null, since: 1, reason: '', pending_handoff: null, epoch: 3 }
+const human: DisplayLease = { ...agent, holder: 'human', viewer_hash: 'abc123abc123', epoch: 4 }
+
+const statusWith = (lease: DisplayLease): DisplayStatus => ({
+ profile: 'ops',
+ profile_key: '/home/hermes/.hermes',
+ supported: true,
+ installed: true,
+ missing: [],
+ running: true,
+ pid: 1,
+ display: ':20',
+ socket: null,
+ geometry: '1440x900',
+ install_command: null,
+ lease
+})
+
+beforeEach(() => $screenState.set({}))
+
+describe('lease epoch ordering', () => {
+ it('a status reply carrying an older epoch does not roll back a newer lease event', () => {
+ setScreenLease(bot, human)
+ setScreenStatus(bot, statusWith(agent))
+
+ expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(human)
+ // The status itself still lands — only its stale lease is ignored.
+ expect(screenStateFor($screenState.get(), bot)?.status?.running).toBe(true)
+ })
+
+ it('a lease event with an older epoch is ignored; a newer or epoch-less one applies', () => {
+ setScreenLease(bot, human)
+ setScreenLease(bot, agent)
+ expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(human)
+
+ const released = { ...agent, epoch: 5 }
+ setScreenLease(bot, released)
+ expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(released)
+
+ const legacy = { ...human, epoch: undefined }
+ setScreenLease(bot, legacy)
+ expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(legacy)
+ })
+})
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.ts
new file mode 100644
index 0000000000000..06bfb8e6f8ddb
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-state.ts
@@ -0,0 +1,89 @@
+/**
+ * Per-bot Bot Screen cache: backend truth (`display.status`) plus the last
+ * `display.lease` event, keyed by the bot's roster identity. Renderer-owned
+ * cache of backend state — never the authority.
+ */
+
+import { atom } from 'nanostores'
+
+import { botSelectionKey } from './data'
+import type { DisplayLease, DisplayStatus, ScreenViewer } from './screen-connection'
+import type { RosterRow } from './types'
+
+export interface BotScreenState {
+ status: DisplayStatus | null
+ lease: DisplayLease | null
+ /** This window's server-minted identity for the bot's current attach; null until the pane observes. */
+ viewer: ScreenViewer | null
+ /** The bot's Hermes has no `display.*` methods (older backend): nothing to check, ever. */
+ unavailable?: boolean
+}
+
+export const $screenState = atom>({})
+
+export function screenStateFor(all: Record, bot: RosterRow): BotScreenState | null {
+ return all[botSelectionKey(bot)] ?? null
+}
+
+/** A lease whose epoch is below the one we hold is a slower response about the
+ * past (a `display.status` reply overtaken by a `display.lease` event). Payloads
+ * without an epoch — older backends — are always applied. */
+function isOlderLease(prev: DisplayLease | null | undefined, next: DisplayLease): boolean {
+ return typeof next.epoch === 'number' && typeof prev?.epoch === 'number' && next.epoch < prev.epoch
+}
+
+export function setScreenStatus(bot: RosterRow, status: DisplayStatus): void {
+ const key = botSelectionKey(bot)
+ const current = $screenState.get()
+ const prev = current[key]
+ const lease = status.lease && !isOlderLease(prev?.lease, status.lease) ? status.lease : (prev?.lease ?? null)
+ $screenState.set({ ...current, [key]: { status, lease, viewer: prev?.viewer ?? null } })
+}
+
+/** `display.status` answered method-not-found: remember it so no surface keeps "checking". */
+export function setScreenUnavailable(bot: RosterRow): void {
+ const key = botSelectionKey(bot)
+ const current = $screenState.get()
+ const prev = current[key]
+
+ if (prev?.unavailable) {
+ return
+ }
+
+ $screenState.set({ ...current, [key]: { status: null, lease: null, viewer: null, unavailable: true } })
+}
+
+export function setScreenLease(bot: RosterRow, lease: DisplayLease): void {
+ const key = botSelectionKey(bot)
+ const current = $screenState.get()
+ const prev = current[key]
+
+ if (isOlderLease(prev?.lease, lease)) {
+ return
+ }
+
+ if (
+ prev?.lease &&
+ prev.lease.holder === lease.holder &&
+ prev.lease.viewer_id === lease.viewer_id &&
+ prev.lease.viewer_hash === lease.viewer_hash &&
+ prev.lease.pending_handoff === lease.pending_handoff
+ ) {
+ return
+ }
+
+ $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease, viewer: prev?.viewer ?? null } })
+}
+
+/** Record the identity `display.observe` minted for this window's attach to `bot`. */
+export function setScreenViewer(bot: RosterRow, viewer: ScreenViewer | null): void {
+ const key = botSelectionKey(bot)
+ const current = $screenState.get()
+ const prev = current[key]
+
+ if ((prev?.viewer ?? null) === viewer) {
+ return
+ }
+
+ $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease: prev?.lease ?? null, viewer } })
+}
diff --git a/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx
new file mode 100644
index 0000000000000..04a782af7eb9d
--- /dev/null
+++ b/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx
@@ -0,0 +1,153 @@
+/**
+ * Viewer identity is SERVER-MINTED: `display.observe` returns the id this attach
+ * is known by, and lease payloads name the holder by `viewer_hash`
+ * (sha256(viewer_id)[:12]) rather than the raw id. "I hold" must be derived
+ * from the minted id, never from a client-generated constant — otherwise a
+ * Desktop reload could claim (or lose) control it does not have.
+ */
+
+import { act, render, waitFor } from '@testing-library/react'
+import { afterEach, beforeEach, expect, it, vi } from 'vitest'
+
+import type { DisplayStatus } from './screen-connection'
+import type * as ScreenConnection from './screen-connection'
+import type { RosterRow } from './types'
+
+vi.mock('@hermes/plugin-sdk', async () => {
+ const { useStore } = await import('@nanostores/react')
+ const { onGatewayEvent } = await import('../../contrib/events')
+
+ return {
+ Button: ({ children, ...props }: React.ButtonHTMLAttributes) => ,
+ Codicon: () => null,
+ GlyphSpinner: () => null,
+ EmptyState: () => null,
+ useValue: useStore,
+ host: { onEvent: onGatewayEvent }
+ }
+})
+vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name }))
+vi.mock('./i18n', () => ({
+ useBots: () => ({
+ screen: {
+ title: 'Screen',
+ youControl: 'You control',
+ otherControls: 'Other controls',
+ agentControls: 'Bot controls',
+ handBack: 'Hand back',
+ handBackForce: 'Hand back (force)',
+ handBackForceHint: 'Force',
+ takeOver: 'Take over',
+ reconnect: 'Reconnect',
+ streamLost: 'Stream lost'
+ }
+ })
+}))
+vi.mock('./screen-connection', async importActual => ({
+ ...(await importActual()),
+ displayRequest: vi.fn(),
+ resolveScreenWsUrl: vi.fn(async () => 'ws://localhost/api/display/ws'),
+ isEventForBotScreen: () => true
+}))
+vi.mock('@novnc/novnc', () => ({
+ default: class {
+ addEventListener() {}
+ disconnect() {}
+ focus() {}
+ }
+}))
+
+// Real event bus, so the pane's listener path is the one under test.
+// eslint-disable-next-line no-restricted-imports
+import { emitGatewayEvent } from '../../contrib/events'
+
+import { displayRequest, viewerHash } from './screen-connection'
+import { BotScreenPane } from './screen-pane'
+import { $screenState } from './screen-state'
+
+const bot: RosterRow = { name: 'default' }
+const MINTED = 'srv-viewer-0001'
+
+const status: DisplayStatus = {
+ profile: 'default',
+ profile_key: '/home/hermes/.hermes',
+ supported: true,
+ installed: true,
+ missing: [],
+ running: true,
+ pid: 42,
+ display: ':20',
+ socket: '/tmp/rfb.sock',
+ geometry: '1440x900',
+ install_command: null,
+ lease: { holder: 'agent', viewer_id: null, viewer_hash: null, pending_handoff: null, since: 1, reason: '' }
+}
+
+beforeEach(() => {
+ $screenState.set({})
+ vi.mocked(displayRequest)
+ .mockReset()
+ .mockImplementation(async (_bot, method) =>
+ method === 'display.observe' ? { ...status, ticket: 'test-ticket', viewer_id: MINTED } : status
+ )
+ vi.stubGlobal(
+ 'WebSocket',
+ class {
+ binaryType = ''
+ close() {}
+ }
+ )
+})
+
+afterEach(() => vi.unstubAllGlobals())
+
+const emitLease = (viewer_hash: string) =>
+ act(() =>
+ emitGatewayEvent({
+ type: 'display.lease',
+ payload: { profile_key: status.profile_key, lease: { ...status.lease, holder: 'human', viewer_hash } }
+ })
+ )
+
+it('holds control when the lease names the hash of the server-minted viewer id, not when it names another', async () => {
+ const view = render()
+ await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe'))
+ await act(async () => {})
+
+ emitLease(await viewerHash('someone-else'))
+ expect(view.queryByText('You control')).toBeNull()
+ expect(view.getByText('Other controls')).toBeTruthy()
+
+ emitLease(await viewerHash(MINTED))
+ expect(view.getByText('You control')).toBeTruthy()
+ view.unmount()
+})
+
+it('hands back with the minted id, never a client-generated one', async () => {
+ const view = render()
+ await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe'))
+ await act(async () => {})
+ emitLease(await viewerHash(MINTED))
+
+ await act(async () => {
+ view.getByText('Hand back').click()
+ })
+ expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.lease.release', { viewer_id: MINTED })
+ view.unmount()
+})
+
+it('offers a forced hand-back for a human lease this window does not hold, sending {force: true} and no viewer id', async () => {
+ const view = render()
+ await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe'))
+ await act(async () => {})
+
+ emitLease(await viewerHash(MINTED))
+ expect(view.queryByText('Hand back (force)')).toBeNull()
+
+ emitLease(await viewerHash('viewer-from-before-the-reload'))
+ await act(async () => {
+ view.getByText('Hand back (force)').click()
+ })
+ expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.lease.release', { force: true })
+ view.unmount()
+})
diff --git a/apps/desktop/src/sdk/index.ts b/apps/desktop/src/sdk/index.ts
index 4ce472aa508dc..39ca078d0e211 100644
--- a/apps/desktop/src/sdk/index.ts
+++ b/apps/desktop/src/sdk/index.ts
@@ -1501,7 +1501,15 @@ export {
PanelRowMenu,
PanelSectionLabel
} from '@/app/overlays/panel'
-export { type RouteContribution, ROUTES_AREA, SIDEBAR_NAV_AREA, type SidebarNavContribution } from '@/app/routes'
+export {
+ type ProfileGroupHeaderContribution,
+ type ProfileGroupRoute,
+ type RouteContribution,
+ ROUTES_AREA,
+ SIDEBAR_NAV_AREA,
+ SIDEBAR_PROFILE_GROUP_HEADER_AREA,
+ type SidebarNavContribution
+} from '@/app/routes'
/** THE full per-toolset config panel core Settings renders — provider picker,
* env vars / API keys, model catalog picker, and post-setup runners. Route-
@@ -1721,6 +1729,9 @@ export const TITLEBAR_AREAS = { center: 'titleBar.center', left: 'titleBar.left'
* setup.runtime_check, reconciled) — pass `host.request`. Don't hand-roll
* readiness from raw RPC shapes. */
export { evaluateRuntimeReadiness, type RuntimeReadinessResult } from '@/lib/runtime-readiness'
+/** A sibling WebSocket beside the route's `/api/ws` (voice PCM, Bot Screen RFB):
+ * same origin, same auth resolution as chat. */
+export { resolveSiblingWsUrl, type SiblingWsRoute } from '@/lib/sibling-ws-url'
/** Canonical time formatting — every surface pulls from here so timestamps read
* the same app-wide. For a row's AGE, bucket with `coarseElapsed` and render
* the compact suffixes (`t.sidebar.row.ageMin` → "52m"), which is what the
diff --git a/apps/desktop/src/store/prompts.ts b/apps/desktop/src/store/prompts.ts
index 7325c4f54ed85..e08c458469716 100644
--- a/apps/desktop/src/store/prompts.ts
+++ b/apps/desktop/src/store/prompts.ts
@@ -98,6 +98,15 @@ interface PendingApprovalPayload {
export interface SudoRequest extends KeyedPrompt {
requestId: string
+ /** JSON-RPC method that resolves this request; the terminal tool's `sudo.respond` by default,
+ * `display.install.sudo.respond` for a Bot Screen package install. */
+ respondMethod?: string
+ /** Description override so the card can say WHAT the password is for. */
+ description?: string
+ /** Immutable origin of the request. The reply is sent to THIS socket, never to whichever
+ * connection happens to be in the foreground when the user presses Send — a password typed for
+ * host A must not travel to host B. */
+ origin?: { connectionId: null | string; profile: string }
}
export interface SecretRequest extends KeyedPrompt {
@@ -225,8 +234,10 @@ export async function replayPendingApproval(gateway: ApprovalGateway | null, ses
* active-session `$*Request` views (same map, fixed key). */
export const sessionApprovalRequest = (sessionId: string | null) =>
computed(approval.$all, all => all[keyFor(sessionId)] ?? null)
+/** A session's sudo card, else the app-level one (a Bot Screen package install is raised with no
+ * session: it belongs to the connection, not to a turn, so whichever chat is focused shows it). */
export const sessionSudoRequest = (sessionId: string | null) =>
- computed(sudo.$all, all => all[keyFor(sessionId)] ?? null)
+ computed(sudo.$all, all => all[keyFor(sessionId)] ?? (sessionId ? all[keyFor(null)] ?? null : null))
export const sessionSecretRequest = (sessionId: string | null) =>
computed(secret.$all, all => all[keyFor(sessionId)] ?? null)
diff --git a/apps/desktop/src/vite-env.d.ts b/apps/desktop/src/vite-env.d.ts
index 11f02fe2a0061..ad5fecb77017e 100644
--- a/apps/desktop/src/vite-env.d.ts
+++ b/apps/desktop/src/vite-env.d.ts
@@ -1 +1,23 @@
///
+
+// @novnc/novnc ships no typings (its export is core/rfb.js); the surface the Bot Screen pane uses.
+// Declared here because `apps/desktop/src/**/*.d.ts` is gitignored except for the allowlisted files,
+// and an ambient `declare module` only works in a script-scoped (import-free) declaration file.
+declare module '@novnc/novnc' {
+ export default class RFB {
+ constructor(target: HTMLElement, urlOrChannel: string | WebSocket | RTCDataChannel, options?: Record)
+ viewOnly: boolean
+ scaleViewport: boolean
+ resizeSession: boolean
+ focusOnClick: boolean
+ background: string
+ qualityLevel: number
+ compressionLevel: number
+ addEventListener(type: string, listener: (event: CustomEvent) => void): void
+ removeEventListener(type: string, listener: (event: CustomEvent) => void): void
+ disconnect(): void
+ focus(): void
+ blur(): void
+ clipboardPasteFrom(text: string): void
+ }
+}
diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py
index 7f850a5423648..6216261a534dc 100644
--- a/hermes_cli/config_defaults.py
+++ b/hermes_cli/config_defaults.py
@@ -2242,6 +2242,15 @@ def _aux(timeout, *, reasoning_effort=True, **extra):
"paste_collapse_threshold_fallback": 5,
"paste_collapse_char_threshold": 2000,
+ # Bot Desktop: a headless Xfce screen per profile on the gateway host (Linux), streamed to Hermes
+ # Desktop where a human can watch, take over (logins, 2FA, CAPTCHAs) and hand back. `hermes computer-use screen`.
+ "bot_desktop": {
+ "geometry": "1440x900",
+ # Opt-in: start the screen automatically the first time computer_use needs a display on a headless
+ # host. Off by default so installing TigerVNC for other reasons never yields a screen nobody asked
+ # for; Hermes Desktop's Screen pane offers Start and this toggle.
+ "auto_start": False,
+ },
"computer_use": {
# cua-driver's upstream PostHog telemetry defaults ON; Hermes sets
# CUA_DRIVER_RS_TELEMETRY_ENABLED=0 in every child env unless this is true.
diff --git a/hermes_cli/dashboard_auth/ws_tickets.py b/hermes_cli/dashboard_auth/ws_tickets.py
index aef1456b84e1f..65391ae239f84 100644
--- a/hermes_cli/dashboard_auth/ws_tickets.py
+++ b/hermes_cli/dashboard_auth/ws_tickets.py
@@ -33,11 +33,13 @@ class TicketInvalid(Exception):
"""Ticket missing, expired, or already consumed."""
-def mint_ticket(*, user_id: str, provider: str) -> str:
+def mint_ticket(*, user_id: str, provider: str, extra: Optional[Dict[str, Any]] = None) -> str:
"""One-shot base64url ticket (32 random bytes) bound to this identity; ``consume_ticket``
- hands the ``info`` dict back to the WS handler."""
+ hands the ``info`` dict back to the WS handler. ``extra`` rides along for routes that need
+ server-chosen context (the Bot Desktop bridge pins the RFB socket's profile home here so a
+ client can never pick another profile's screen)."""
ticket = secrets.token_urlsafe(32)
- info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time())}
+ info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time()), **(extra or {})}
with _lock:
_tickets[ticket] = (int(time.time()) + TTL_SECONDS, info)
_gc_expired_locked()
diff --git a/hermes_cli/pty_bridge.py b/hermes_cli/pty_bridge.py
index 90d6a61a6c338..2788eb9ecee12 100644
--- a/hermes_cli/pty_bridge.py
+++ b/hermes_cli/pty_bridge.py
@@ -10,13 +10,13 @@
import asyncio
import errno
-import fcntl
+import fcntl # windows-footgun: ok — POSIX-only module by design (see docstring)
import os
import select
import signal
import struct
import sys
-import termios
+import termios # windows-footgun: ok — POSIX-only module by design (see docstring)
import time
from typing import Optional, Sequence
diff --git a/hermes_cli/subcommands/computer_use.py b/hermes_cli/subcommands/computer_use.py
index ac0d52f4b88ad..22e0effcd6daa 100644
--- a/hermes_cli/subcommands/computer_use.py
+++ b/hermes_cli/subcommands/computer_use.py
@@ -174,6 +174,12 @@ def build_computer_use_parser(subparsers) -> None:
"grant", help="Request the grants (opens the dialog attributed to CuaDriver)")
_perms_actions = {"grant": _cu_perms_grant, "status": _cu_perms_status}
+ from hermes_cli.subcommands.computer_use_screen import build_screen_parser
+ build_screen_parser(computer_use_sub, add_json_flag)
+
+ def _cu_screen(args):
+ return args.screen_func(args)
+
def _cu_permissions(args):
handler = _perms_actions.get(getattr(args, "computer_use_perms_action", None))
if handler is not None:
@@ -181,7 +187,7 @@ def _cu_permissions(args):
computer_use_perms.print_help()
_actions = {"install": _cu_install, "status": _cu_status, "doctor": _cu_doctor,
- "permissions": _cu_permissions}
+ "permissions": _cu_permissions, "screen": _cu_screen}
def cmd_computer_use(args):
handler = _actions.get(getattr(args, "computer_use_action", None))
diff --git a/hermes_cli/subcommands/computer_use_screen.py b/hermes_cli/subcommands/computer_use_screen.py
new file mode 100644
index 0000000000000..5e76b521f9918
--- /dev/null
+++ b/hermes_cli/subcommands/computer_use_screen.py
@@ -0,0 +1,117 @@
+"""``hermes computer-use screen`` — the Bot Desktop screen a profile's ``computer_use`` drives on a
+headless Linux gateway host, viewable from Hermes Desktop. ``status`` / ``start`` / ``stop`` /
+``install`` mirror the Desktop pane's controls for ops shells and cloud images."""
+
+from __future__ import annotations
+
+import getpass
+import json
+import sys
+
+
+def _screen_status(args) -> int:
+ from tools.bot_desktop import lease, runtime
+ st = runtime.status()
+ if bool(getattr(args, "json", False)):
+ print(json.dumps({**st.as_dict(), "lease": lease.get().as_dict()}, indent=2, sort_keys=True))
+ return 0 if st.running else 1
+ if not st.supported:
+ print("Bot Desktop screens run on Linux gateway hosts only (this host keeps its real display).")
+ return 1
+ if not st.installed:
+ print("Bot Desktop: packages missing → " + ", ".join(st.missing))
+ print(" Install: " + (st.install_command or "hermes computer-use screen install"))
+ return 1
+ if st.running:
+ holder = lease.get()
+ who = f"human ({holder.viewer_id})" if holder.holder == "human" else "agent"
+ print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry}), pid {st.pid}")
+ print(f" control: {who} rfb socket: {st.socket}")
+ print(" View it: Hermes Desktop → Bots → this bot → Screen")
+ return 0
+ print(f"Bot Desktop [{st.profile}]: installed, not running. Start: hermes computer-use screen start")
+ return 1
+
+
+def _screen_start(args) -> int:
+ from tools.bot_desktop import runtime
+ try:
+ st = runtime.start()
+ except RuntimeError as exc:
+ print(f"Bot Desktop: {exc}")
+ return 1
+ print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry})")
+ return 0
+
+
+def _screen_stop(args) -> int:
+ from tools.bot_desktop import runtime
+ if runtime.is_supported_host():
+ from tools.bot_desktop import lease
+ lease.release()
+ print("Bot Desktop: stopped" if runtime.stop() else "Bot Desktop: was not running")
+ return 0
+
+
+def _screen_install(args) -> int:
+ from tools.bot_desktop import install, runtime
+ if not runtime.is_supported_host():
+ print("Bot Desktop screens run on Linux gateway hosts only.")
+ return 1
+ if not runtime.missing_binaries():
+ print("Bot Desktop: packages already installed.")
+ return 0
+ cmd = runtime.install_command()
+ if cmd is None:
+ print("Bot Desktop: no supported package manager (apt/dnf/pacman) found. Install TigerVNC (Xvnc) and "
+ "the Xfce core (xfwm4, xfce4-panel, xfdesktop, xfce4-settings) by hand.")
+ return 1
+ print(f"Bot Desktop: installing → {cmd}")
+ if not bool(getattr(args, "yes", False)) and sys.stdin.isatty():
+ answer = input("Proceed? [Y/n] ").strip().lower()
+ if answer not in ("", "y", "yes"):
+ return 1
+ # Same runner as the Desktop pane's Install button: one per-profile slot, list-form spawn, sudo password
+ # via stdin (-S) and never on the command line.
+ try:
+ rc = install.install_packages(ask_password=lambda: getpass.getpass("[sudo] password: "), on_line=print)
+ except install.InstallBusy as exc:
+ print(f"Bot Desktop: {exc}")
+ return 1
+ if rc != 0:
+ print(f"Bot Desktop: installer exited {rc}")
+ return rc or 1
+ missing = runtime.missing_binaries()
+ if missing:
+ print("Bot Desktop: still missing " + ", ".join(missing))
+ return 1
+ print("Bot Desktop: ready. Start with `hermes computer-use screen start` or from Hermes Desktop.")
+ return 0
+
+
+SCREEN_ACTIONS = {"status": _screen_status, "start": _screen_start, "stop": _screen_stop, "install": _screen_install}
+
+
+def build_screen_parser(computer_use_sub, add_json_flag) -> None:
+ screen = computer_use_sub.add_parser(
+ "screen", help="Bot Desktop: the headless screen this profile's computer_use drives (Linux)",
+ description="On a headless Linux gateway host Hermes gives each profile its own Xfce screen\n"
+ "(TigerVNC Xvnc on a private Unix socket). The agent's computer_use and headed\n"
+ "browser act on it; Hermes Desktop shows it live and lets a human take over for\n"
+ "logins, 2FA or CAPTCHAs, then hand control back.\n\n"
+ "`install` adds the system packages (apt/dnf/pacman); `start`/`stop` manage this\n"
+ "profile's screen; `status` shows display, control holder and socket.")
+ sub = screen.add_subparsers(dest="computer_use_screen_action")
+ st = sub.add_parser("status", help="Show whether this profile's screen is installed/running and who holds control")
+ add_json_flag(st, "Emit the status payload as JSON.")
+ sub.add_parser("start", help="Start this profile's screen")
+ sub.add_parser("stop", help="Stop this profile's screen (hands control back to the agent first)")
+ inst = sub.add_parser("install", help="Install TigerVNC + Xfce core via the host package manager")
+ inst.add_argument("-y", "--yes", action="store_true", help="Do not ask before running the package manager")
+
+ def _cmd(args):
+ handler = SCREEN_ACTIONS.get(str(getattr(args, "computer_use_screen_action", None) or ""))
+ if handler is not None:
+ return handler(args)
+ screen.print_help()
+ screen.set_defaults(screen_func=_cmd)
diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py
new file mode 100644
index 0000000000000..4ccb2721893fc
--- /dev/null
+++ b/hermes_cli/web_routers/display.py
@@ -0,0 +1,184 @@
+"""``/api/display/ws`` — raw RFB over WebSocket for the Bot Desktop viewer.
+
+The Desktop renderer calls ``display.observe`` on its authenticated ``/api/ws`` connection, gets a
+single-use 30 s ticket pinned to that profile's RFB socket, then opens this route with
+``?display_ticket=``. No websockify, no new port: the bridge splices the profile's 0600 Unix socket
+into the WebSocket as binary frames with backpressure both ways, and runs the client stream through
+:class:`tools.bot_desktop.rfb_filter.RfbClientFilter` so keyboard, pointer and clipboard reach Xvnc
+only from the viewer that currently holds the lease. noVNC's ``viewOnly`` is UX; this is the gate.
+
+A lease change closes the evicted viewer's socket with 4000 ``control-taken`` so its UI drops back to
+Watch mode and reconnects.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Optional
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from hermes_cli.web_server_chat import _ws_request_is_allowed
+
+_log = logging.getLogger(__name__)
+router = APIRouter()
+
+_READ_CHUNK = 64 * 1024
+_CLOSE_CONTROL_TAKEN = 4000
+_CLEAN_CLOSE = frozenset({1000, 1001})
+_CLOSE_DESKTOP_GONE = 4001
+_CLOSE_BAD_TICKET = 4401
+_CLOSE_NOT_ALLOWED = 4403
+_CLOSE_PROTOCOL = 1003
+_LEASE_REFRESH_S = 0.25
+
+
+def _should_evict(held: dict, lease, viewer_id: str) -> bool:
+ """A viewer that held control during this connection and lost it to ANOTHER human is kicked so its
+ UI repaints; a plain hand-back to the agent, pure watchers and the new holder stay connected. The
+ hand-back also forgets that this viewer ever held: after it, they are a plain watcher again and a
+ later takeover by someone else must not evict them. ``held`` is the per-connection memory."""
+ from tools.bot_desktop import lease as _lease
+ if lease.holder != _lease.HUMAN:
+ held["ever"] = False
+ return False
+ if lease.viewer_id == viewer_id:
+ held["ever"] = True
+ return False
+ return bool(held["ever"])
+
+
+def _consume_display_ticket(ws: WebSocket) -> Optional[dict]:
+ from hermes_cli.dashboard_auth.ws_tickets import TicketInvalid, consume_ticket
+ ticket = ws.query_params.get("display_ticket", "")
+ if not ticket:
+ return None
+ try:
+ info = consume_ticket(ticket)
+ except TicketInvalid:
+ return None
+ if info.get("provider") != "bot-desktop" or not info.get("hermes_home"):
+ return None
+ return info
+
+
+@router.websocket("/api/display/ws")
+async def display_ws(ws: WebSocket) -> None:
+ if not _ws_request_is_allowed(ws):
+ await ws.close(code=_CLOSE_NOT_ALLOWED)
+ return
+ info = _consume_display_ticket(ws)
+ if info is None:
+ await ws.close(code=_CLOSE_BAD_TICKET, reason="display ticket missing, expired or used")
+ return
+ await _bridge(ws, info)
+
+
+async def _bridge(ws: WebSocket, info: dict) -> None:
+ """Pump RFB bytes between the viewer socket and THIS profile's Xvnc, gated by the lease."""
+ from hermes_constants import hermes_home_key
+ from tools.bot_desktop import lease as _lease
+ from tools.bot_desktop.rfb_filter import RfbClientFilter
+ from pathlib import Path
+
+ sock = Path(info["hermes_home"]) / "bot-desktop" / "rfb.sock"
+ profile_home = str(info["hermes_home"])
+ profile_key = hermes_home_key(profile_home)
+ viewer_id = str(info.get("viewer_id") or info.get("user_id") or "viewer")
+ if not sock.exists():
+ await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop is not running")
+ return
+ try:
+ reader, writer = await asyncio.open_unix_connection(str(sock))
+ except OSError as exc:
+ _log.warning("display ws: cannot reach RFB socket %s: %s", sock, exc)
+ await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop socket unreachable")
+ return
+
+ await ws.accept()
+ loop = asyncio.get_running_loop()
+ evicted = asyncio.Event()
+ held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)}
+ # Input gate cache: reading lease.json per client message (a stat + read on the event loop for
+ # every pointer move) is replaced by a decision refreshed on this process's on_change callback
+ # and by a file re-read at most every _LEASE_REFRESH_S, so another process's takeover still lands.
+ allowed = {"input": held["ever"], "at": loop.time()}
+
+ def _refresh_allowed(lease=None) -> None:
+ if lease is None:
+ lease = _lease.get(profile_key=profile_home)
+ allowed["input"] = lease.holder == _lease.HUMAN and lease.viewer_id == viewer_id
+ allowed["at"] = loop.time()
+
+ def _may_send_input() -> bool:
+ if loop.time() - allowed["at"] > _LEASE_REFRESH_S:
+ _refresh_allowed()
+ return allowed["input"]
+
+ def _on_lease(key: str, lease) -> None:
+ if key != profile_key:
+ return
+ loop.call_soon_threadsafe(_refresh_allowed, lease)
+ if _should_evict(held, lease, viewer_id):
+ loop.call_soon_threadsafe(evicted.set)
+ unsubscribe = _lease.on_change(_on_lease)
+
+ rfb_filter = RfbClientFilter(_may_send_input)
+
+ viewer_closed = asyncio.Event()
+
+ async def rfb_to_ws() -> None:
+ while True:
+ chunk = await reader.read(_READ_CHUNK)
+ if not chunk:
+ return
+ await ws.send_bytes(chunk) # awaiting the send is the backpressure toward Xvnc
+
+ async def ws_to_rfb() -> None:
+ while True:
+ message = await ws.receive()
+ if message.get("type") == "websocket.disconnect":
+ # 1000/1001 = the viewer closed the window; anything else is a dropped link.
+ if message.get("code") in _CLEAN_CLOSE:
+ viewer_closed.set()
+ return
+ data = message.get("bytes")
+ if data is None:
+ await ws.close(code=_CLOSE_PROTOCOL, reason="RFB is binary")
+ return
+ try:
+ allowed = rfb_filter.feed(data)
+ except ValueError as exc:
+ await ws.close(code=_CLOSE_PROTOCOL, reason=str(exc)[:100])
+ return
+ if allowed:
+ writer.write(allowed)
+ await writer.drain() # backpressure toward the browser
+
+ async def watch_eviction() -> None:
+ await evicted.wait()
+ await ws.close(code=_CLOSE_CONTROL_TAKEN, reason="control-taken")
+
+ tasks = [asyncio.create_task(rfb_to_ws()), asyncio.create_task(ws_to_rfb()),
+ asyncio.create_task(watch_eviction())]
+ try:
+ done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ for t in pending:
+ t.cancel()
+ for t in done:
+ exc = t.exception()
+ if exc and not isinstance(exc, (WebSocketDisconnect, ConnectionError)):
+ _log.debug("display ws ended: %r", exc)
+ finally:
+ unsubscribe()
+ writer.close()
+ # Closing the viewer window hands control back. A DROPPED link (laptop lid, Wi-Fi, 1006)
+ # keeps the human's exclusion: they may be mid-login on that screen and the agent must not
+ # resume into it. The Desktop reconnects into the same lease, or the human hands back.
+ if viewer_closed.is_set() and _lease.viewer_may_send_input(viewer_id, profile_key=profile_home):
+ _lease.release(viewer_id, profile_key=profile_home)
+ try:
+ await ws.close()
+ except Exception: # already closed by the peer or by an eviction
+ pass
diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py
index d95f8182a6968..73faef43e6b35 100644
--- a/hermes_cli/web_server.py
+++ b/hermes_cli/web_server.py
@@ -932,6 +932,7 @@ def _get_dashboard_plugins(force_rescan: bool = False) -> list:
status as _status_routes,
actions as _actions_routes,
audio as _audio_routes,
+ display as _display_routes,
sessions as _sessions_routes,
profiles as _profiles_routes,
memory_providers as _memory_providers_routes,
@@ -955,6 +956,7 @@ def _get_dashboard_plugins(force_rescan: bool = False) -> list:
app.include_router(_status_routes.router)
app.include_router(_actions_routes.router)
app.include_router(_audio_routes.router)
+app.include_router(_display_routes.router)
app.include_router(_actions_routes.status_router)
app.include_router(_sessions_routes.list_router)
app.include_router(_profiles_routes.sessions_router)
diff --git a/hermes_cli/web_server_chat.py b/hermes_cli/web_server_chat.py
index 047bc867afb9a..7196a3254f451 100644
--- a/hermes_cli/web_server_chat.py
+++ b/hermes_cli/web_server_chat.py
@@ -270,7 +270,12 @@ def _stamp_identity(info) -> None:
return "no_credential", "none"
try:
- _stamp_identity(consume_ticket(ticket))
+ info = consume_ticket(ticket)
+ if info.get("provider") == "bot-desktop":
+ # A display ticket admits one RFB bridge on /api/display/ws (a watch-only
+ # capability handed to a screen viewer); it must not double as a login here.
+ raise TicketInvalid("display ticket presented as a gateway login")
+ _stamp_identity(info)
if protocol_ticket:
# Select only the stable public protocol during accept. The
# ticket-bearing protocol is a credential and must never be
diff --git a/package-lock.json b/package-lock.json
index eaf154d450768..a24c4496ef6a1 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -31,7 +31,7 @@
},
"apps/bootstrap-installer": {
"name": "@hermes/bootstrap-installer",
- "version": "0.0.1",
+ "version": "0.21.1",
"dependencies": {
"@nous-research/ui": "0.18.2",
"@tailwindcss/typography": "0.5.20",
@@ -85,6 +85,7 @@
"@lezer/highlight": "1.2.3",
"@nanostores/react": "1.1.0",
"@nous-research/ui": "0.18.2",
+ "@novnc/novnc": "1.7.0",
"@streamdown/code": "1.1.1",
"@streamdown/math": "1.0.2",
"@tabler/icons-react": "3.44.0",
@@ -3498,6 +3499,12 @@
}
}
},
+ "node_modules/@novnc/novnc": {
+ "version": "1.7.0",
+ "resolved": "https://registry.npmjs.org/@novnc/novnc/-/novnc-1.7.0.tgz",
+ "integrity": "sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==",
+ "license": "MPL-2.0"
+ },
"node_modules/@npmcli/agent": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.2.2.tgz",
diff --git a/pyproject.toml b/pyproject.toml
index ee77c1e53a9b4..7deda29005ea2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -578,6 +578,9 @@ gateway = ["assets/**/*"]
# sealed wheels with the plugin Python modules; without this declaration the
# wheel contains adapters but discovery finds zero bundled plugins.
plugins = ["**/plugin.yaml", "**/plugin.yml"]
+# Bot Desktop starts Xvnc + Xfce through a shell launcher and seeds the wallpaper; both are read
+# via Path(__file__).parent in tools/bot_desktop/runtime.py and vanish from sealed wheels otherwise.
+tools = ["bot_desktop/launcher.sh", "bot_desktop/wallpaper.png"]
[tool.pytest.ini_options]
testpaths = ["tests"]
diff --git a/scripts/check-windows-footguns.py b/scripts/check-windows-footguns.py
index 30100402d0ad0..d5de92b410c12 100644
--- a/scripts/check-windows-footguns.py
+++ b/scripts/check-windows-footguns.py
@@ -431,6 +431,27 @@ class Footgun:
and _call_closes_on_line(line, m.end())
),
),
+ Footgun(
+ name="module-level import of a POSIX-only stdlib module",
+ # Only unindented imports: a top-level `import fcntl` fails at import time on Windows and takes
+ # every importer down with it (tools.bot_desktop.lease took computer_use down on native
+ # Windows). Indented imports inside a function or a try/except ImportError are the fix shape.
+ pattern=re.compile(
+ r"^(?:import\s+(?:fcntl|pwd|grp|termios|resource|pty|tty)\b"
+ r"|from\s+(?:fcntl|pwd|grp|termios|resource|pty|tty)\s+import\b)"
+ ),
+ message=(
+ "fcntl/pwd/grp/termios/resource/pty/tty do not exist on Windows; a module-level import "
+ "raises ModuleNotFoundError and breaks every module that imports this one."
+ ),
+ fix=(
+ "Import lazily inside the function that needs it, or\n"
+ "try:\n"
+ " import fcntl\n"
+ "except ImportError:\n"
+ " fcntl = None # and take the Windows path when None"
+ ),
+ ),
]
diff --git a/scripts/profile-tui.py b/scripts/profile-tui.py
index 4d86aa057bfa9..a59b76659d822 100755
--- a/scripts/profile-tui.py
+++ b/scripts/profile-tui.py
@@ -26,7 +26,7 @@
import argparse
import json
import os
-import pty
+import pty # windows-footgun: ok — dev profiling script, POSIX pty by design
import select
import signal
import sqlite3
diff --git a/tests/hermes_cli/test_bot_desktop_screen_stop.py b/tests/hermes_cli/test_bot_desktop_screen_stop.py
new file mode 100644
index 0000000000000..33f661aed5077
--- /dev/null
+++ b/tests/hermes_cli/test_bot_desktop_screen_stop.py
@@ -0,0 +1,19 @@
+"""The documented CLI stop is also recovery for a disconnected viewer's lease."""
+
+import argparse
+
+from hermes_cli.subcommands.computer_use_screen import build_screen_parser
+import pytest
+
+
+@pytest.mark.linux_only
+def test_screen_stop_hands_back_even_when_the_desktop_has_already_exited():
+ from tools.bot_desktop import lease
+
+ parser = argparse.ArgumentParser()
+ build_screen_parser(parser.add_subparsers(), lambda sub, help_text: sub.add_argument('--json', action='store_true'))
+ lease.acquire('disconnected-viewer')
+ args = parser.parse_args(['screen', 'stop'])
+ assert args.screen_func(args) == 0
+ assert lease.get().holder == lease.AGENT
+ assert lease.wait_for_release(timeout=0)
diff --git a/tests/hermes_cli/test_computer_use_screen_install.py b/tests/hermes_cli/test_computer_use_screen_install.py
new file mode 100644
index 0000000000000..a0199600801c2
--- /dev/null
+++ b/tests/hermes_cli/test_computer_use_screen_install.py
@@ -0,0 +1,29 @@
+"""`hermes computer-use screen install` shares the Desktop pane's installer (one lock, list-form spawn)."""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+
+from hermes_cli.subcommands.computer_use_screen import build_screen_parser
+from tools.bot_desktop import install, runtime
+
+
+def test_cli_install_goes_through_the_shared_installer(monkeypatch):
+ monkeypatch.setattr(runtime, "is_supported_host", lambda: True)
+ monkeypatch.setattr(runtime, "missing_binaries", lambda: ["Xvnc"])
+ monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y tigervnc-standalone-server")
+ monkeypatch.setattr(subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(AssertionError("shell install spawned")))
+ calls = []
+
+ def fake_install(*, ask_password, on_line, **kw):
+ calls.append((callable(ask_password), callable(on_line)))
+ return 0
+
+ monkeypatch.setattr(install, "install_packages", fake_install)
+ monkeypatch.setattr(runtime, "missing_binaries", lambda: ["Xvnc"] if not calls else [])
+ parser = argparse.ArgumentParser()
+ build_screen_parser(parser.add_subparsers(), lambda sub, help_text: sub.add_argument("--json", action="store_true"))
+ args = parser.parse_args(["screen", "install", "-y"])
+ assert args.screen_func(args) == 0
+ assert calls == [(True, True)]
diff --git a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py
new file mode 100644
index 0000000000000..2bc98c15dba05
--- /dev/null
+++ b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py
@@ -0,0 +1,130 @@
+"""A dropped viewer link (1006: lid closed, Wi-Fi) must NOT hand the screen back to the agent — the
+human may be mid-login on it. Only a clean close (1000/1001) releases."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import tempfile
+
+import pytest
+
+from hermes_cli.web_routers import display
+from tools.bot_desktop import lease
+
+
+class _Ws:
+ """Just enough of a Starlette WebSocket: one disconnect message with the given close code."""
+
+ def __init__(self, close_code: int):
+ self._code = close_code
+ self.closed = False
+
+ async def accept(self):
+ pass
+
+ async def receive(self):
+ await asyncio.sleep(0.05)
+ return {"type": "websocket.disconnect", "code": self._code}
+
+ async def send_bytes(self, data):
+ pass
+
+ async def close(self, code=1000, reason=""):
+ self.closed = True
+
+
+async def _bridge_once(close_code: int, home: str) -> lease.Lease:
+ sock_dir = os.path.join(home, "bot-desktop")
+ os.makedirs(sock_dir, exist_ok=True)
+ sock = os.path.join(sock_dir, "rfb.sock")
+
+ async def _xvnc(reader, writer): # a silent framebuffer
+ await asyncio.sleep(1)
+ writer.close()
+
+ server = await asyncio.start_unix_server(_xvnc, path=sock)
+ try:
+ info = {"hermes_home": home, "viewer_id": "desk-1"}
+ await display._bridge(_Ws(close_code), info)
+ finally:
+ server.close()
+ return lease.get(profile_key=home)
+
+
+# 1005 (no status code) is what noVNC's code-less socket.close() AND some proxies produce on a drop, so
+# the server keeps the lease; the Desktop sends an explicit 1000 when the pane is closed on purpose.
+@pytest.mark.parametrize(("close_code", "human_keeps_control"), [(1006, True), (1005, True), (1000, False)])
+def test_only_a_clean_viewer_close_hands_the_screen_back(monkeypatch, close_code, human_keeps_control):
+ lease._reset_for_tests()
+ with tempfile.TemporaryDirectory() as home:
+ lease.acquire("desk-1", profile_key=home)
+ after = asyncio.run(_bridge_once(close_code, home))
+ lease._reset_for_tests()
+ assert (after.holder == lease.HUMAN) is human_keeps_control, after
+
+
+def test_ex_holder_who_handed_back_is_not_evicted_by_a_later_takeover():
+ """desk-1 holds, hands back to the agent, then desk-2 takes over: desk-1 is a plain watcher again
+ and must stay connected; only a takeover WHILE desk-1 held (or believed it held) kicks it."""
+ held = {"ever": False}
+ assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-1"), "desk-1") is False
+ assert display._should_evict(held, lease.Lease(holder=lease.AGENT), "desk-1") is False
+ assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-2"), "desk-1") is False
+
+ held = {"ever": False}
+ display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-1"), "desk-1")
+ assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-2"), "desk-1") is True
+
+
+class _OpenWs(_Ws):
+ """Stays open until ``finish`` is set, then reports a clean close."""
+
+ def __init__(self):
+ super().__init__(1000)
+ self.finish = asyncio.Event()
+
+ async def receive(self):
+ await self.finish.wait()
+ return {"type": "websocket.disconnect", "code": self._code}
+
+
+def test_a_takeover_made_by_another_process_stops_input_within_the_refresh_interval(monkeypatch):
+ """The bridge caches the input decision instead of reading lease.json per message; a takeover
+ written by ANOTHER process (no in-process listener fires) must still be seen quickly."""
+ from tools.bot_desktop import rfb_filter
+ captured = {}
+
+ class _Filter(rfb_filter.RfbClientFilter):
+ def __init__(self, allow_input):
+ super().__init__(allow_input)
+ captured["allow"] = allow_input
+ monkeypatch.setattr(rfb_filter, "RfbClientFilter", _Filter)
+
+ async def _run(home: str) -> float:
+ sock_dir = os.path.join(home, "bot-desktop")
+ os.makedirs(sock_dir, exist_ok=True)
+ server = await asyncio.start_unix_server(lambda r, w: None, path=os.path.join(sock_dir, "rfb.sock"))
+ ws = _OpenWs()
+ task = asyncio.create_task(display._bridge(ws, {"hermes_home": home, "viewer_id": "desk-1"}))
+ try:
+ while "allow" not in captured:
+ await asyncio.sleep(0.01)
+ assert captured["allow"]() is True
+ # Another process takes over: the file changes, no listener in this process is told.
+ lease._write(lease._path(home), lease.Lease(holder=lease.HUMAN, viewer_id="desk-2", epoch=2))
+ t0 = asyncio.get_running_loop().time()
+ while captured["allow"]() and asyncio.get_running_loop().time() - t0 < 2.0:
+ await asyncio.sleep(0.02)
+ return asyncio.get_running_loop().time() - t0
+ finally:
+ ws.finish.set()
+ await task
+ server.close()
+
+ lease._reset_for_tests()
+ with tempfile.TemporaryDirectory() as home:
+ lease.acquire("desk-1", profile_key=home)
+ elapsed = asyncio.run(_run(home))
+ lease._reset_for_tests()
+ assert elapsed < 0.5, elapsed
diff --git a/tests/hermes_cli/test_display_ws_ticket.py b/tests/hermes_cli/test_display_ws_ticket.py
new file mode 100644
index 0000000000000..fc072596b15fe
--- /dev/null
+++ b/tests/hermes_cli/test_display_ws_ticket.py
@@ -0,0 +1,27 @@
+"""The display bridge admits only a display ticket minted for THIS profile's socket: a gateway ticket,
+an expired ticket, or one for another provider is refused before any socket is dialled."""
+
+from __future__ import annotations
+
+from hermes_cli.dashboard_auth import ws_tickets
+from hermes_cli.web_routers import display
+
+
+class _Ws:
+ def __init__(self, **params):
+ self.query_params = params
+
+
+def test_display_ticket_must_be_a_bot_desktop_ticket_pinned_to_a_profile_home(monkeypatch):
+ ws_tickets._reset_for_tests()
+ gateway_ticket = ws_tickets.mint_ticket(user_id="u", provider="google")
+ assert display._consume_display_ticket(_Ws(display_ticket=gateway_ticket)) is None
+
+ unpinned = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop")
+ assert display._consume_display_ticket(_Ws(display_ticket=unpinned)) is None
+
+ good = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop",
+ extra={"hermes_home": "/srv/hermes/bot-a", "viewer_id": "v"})
+ info = display._consume_display_ticket(_Ws(display_ticket=good))
+ assert info and info["hermes_home"] == "/srv/hermes/bot-a" and info["viewer_id"] == "v"
+ assert display._consume_display_ticket(_Ws(display_ticket=good)) is None, "single use"
diff --git a/tests/hermes_cli/test_ws_auth_display_ticket.py b/tests/hermes_cli/test_ws_auth_display_ticket.py
new file mode 100644
index 0000000000000..626f2c106c38c
--- /dev/null
+++ b/tests/hermes_cli/test_ws_auth_display_ticket.py
@@ -0,0 +1,37 @@
+"""A Bot Desktop display ticket admits one RFB bridge on ``/api/display/ws`` — it must never pass as a
+full login on ``/api/ws`` (the reverse of ``test_display_ws_ticket``, which refuses gateway tickets there)."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from hermes_cli import web_server
+import hermes_cli.web_server_chat as _web_server_chat
+from hermes_cli.dashboard_auth.ws_tickets import _reset_for_tests, mint_ticket
+
+
+@pytest.fixture
+def gated_state():
+ _reset_for_tests()
+ prev = getattr(web_server.app.state, "auth_required", None)
+ web_server.app.state.auth_required = True
+ yield
+ web_server.app.state.auth_required = prev
+ _reset_for_tests()
+
+
+def _ws(ticket: str):
+ return SimpleNamespace(
+ query_params={"ticket": ticket}, headers={},
+ client=SimpleNamespace(host="203.0.113.9"), url=SimpleNamespace(path="/api/ws"))
+
+
+def test_display_ticket_is_refused_as_a_gateway_login(gated_state):
+ ticket = mint_ticket(user_id="display:v", provider="bot-desktop",
+ extra={"hermes_home": "/srv/hermes/bot-a", "viewer_id": "v"})
+ ws = _ws(ticket)
+ reason, _credential = _web_server_chat._ws_auth_reason(ws)
+ assert reason == "ticket_invalid"
+ assert not hasattr(ws, "_hermes_auth_identity")
diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py
index f5c2c431fd296..34811b799b72f 100644
--- a/tests/tools/conftest.py
+++ b/tests/tools/conftest.py
@@ -35,6 +35,24 @@ def _no_host_browser_use_cli():
yield
+@pytest.fixture(autouse=True)
+def _no_host_bot_desktop_autostart():
+ """Keep the host's TigerVNC/Xfce install out of tests.
+
+ ``computer_use`` auto-starts the profile's Bot Desktop on a headless Linux
+ host with the packages installed, so a developer box that has them would
+ launch a real Xvnc + Xfce session per test. Pin the binaries to "missing";
+ tests that exercise the desktop path monkeypatch ``runtime`` themselves.
+ """
+ try:
+ from tools.bot_desktop import runtime as bd_runtime
+ except Exception:
+ yield
+ return
+ with patch.object(bd_runtime, "missing_binaries", lambda: ["Xvnc"]):
+ yield
+
+
@pytest.fixture(autouse=True)
def _materialize_mcp_sdk_symbols():
"""Materialize the lazily-imported MCP SDK before each tools test.
diff --git a/tests/tools/test_bot_desktop_browser.py b/tests/tools/test_bot_desktop_browser.py
new file mode 100644
index 0000000000000..451702e2c4f32
--- /dev/null
+++ b/tests/tools/test_bot_desktop_browser.py
@@ -0,0 +1,91 @@
+"""The dock's Browser and agent-browser resolve to one identity: same executable, same user-data-dir —
+and when a human opened that browser first, the agent attaches to it instead of launching a second one
+(Chromium's profile singleton would forward the launch and kill it without a DevTools endpoint)."""
+
+from __future__ import annotations
+
+import os
+import socket
+
+from tools.bot_desktop import browser, runtime
+
+
+def test_dock_and_agent_share_browser_identity(tmp_path, monkeypatch):
+ exe = tmp_path / "chrome"
+ exe.write_text("#!/bin/sh\n", encoding="utf-8")
+ exe.chmod(0o755)
+ monkeypatch.setenv("AGENT_BROWSER_EXECUTABLE_PATH", str(exe))
+ monkeypatch.delenv("AGENT_BROWSER_PROFILE", raising=False)
+ monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bot-desktop")
+
+ dock_exe, dock_profile = browser.dock_launch()
+ agent_env = browser.env_for_agent({})
+
+ assert dock_exe == agent_env["AGENT_BROWSER_EXECUTABLE_PATH"] == str(exe)
+ assert dock_profile == agent_env["AGENT_BROWSER_PROFILE"] == str(tmp_path / "bot-desktop" / "browser-profile")
+
+
+def test_user_pinned_profile_wins(tmp_path, monkeypatch):
+ monkeypatch.setenv("AGENT_BROWSER_PROFILE", str(tmp_path / "mine"))
+ monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bot-desktop")
+ assert browser.profile_dir() == tmp_path / "mine"
+
+
+def test_dock_browser_advertises_a_devtools_port():
+ """A human-started instance must be attachable, or the agent can never drive it afterwards."""
+ assert "--remote-debugging-port=" in browser.dock_command("/opt/chrome", "/p/dir").split()[2]
+
+
+def _fake_running_instance(user_data_dir, pid: int, port: int) -> None:
+ (user_data_dir / "DevToolsActivePort").write_text(f"{port}\n/devtools/browser/abc\n", encoding="utf-8")
+ os.symlink(f"host-{pid}", user_data_dir / "SingletonLock")
+
+
+def test_running_instance_port_requires_live_pid_and_open_port(tmp_path):
+ listener = socket.socket()
+ listener.bind(("127.0.0.1", 0))
+ listener.listen(1)
+ port = listener.getsockname()[1]
+ try:
+ _fake_running_instance(tmp_path, os.getpid(), port)
+ assert browser.running_instance_cdp_port(str(tmp_path)) == port
+
+ # Both files outlive a closed Chromium: a dead pid must not be trusted.
+ os.unlink(tmp_path / "SingletonLock")
+ os.symlink("host-2147483000", tmp_path / "SingletonLock")
+ assert browser.running_instance_cdp_port(str(tmp_path)) is None
+ finally:
+ listener.close()
+ # Live pid, port no longer accepting: still not attachable.
+ os.unlink(tmp_path / "SingletonLock")
+ os.symlink(f"host-{os.getpid()}", tmp_path / "SingletonLock")
+ assert browser.running_instance_cdp_port(str(tmp_path)) is None
+ assert browser.running_instance_cdp_port(str(tmp_path / "missing")) is None
+
+
+def test_agent_attaches_to_human_started_browser(monkeypatch):
+ """With a live dock instance on the shared profile the local argv carries ``--cdp ``; without one
+ it stays a plain ``--session`` launch."""
+ from tools import browser_tool_session as session
+
+ monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":37"})
+ monkeypatch.setattr(session._cloud, "_get_browser_engine", lambda: "auto")
+ monkeypatch.setattr(session._cloud, "_is_headed_mode", lambda: False)
+ monkeypatch.setattr(session, "_agent_browser_argv", lambda cmd: [cmd])
+ argvs: list = []
+
+ def spawn(task_id, session_info, cmd_parts, *rest):
+ argvs.append(cmd_parts)
+ return {"success": True}
+
+ monkeypatch.setattr(session, "_spawn_and_collect", spawn)
+ monkeypatch.setattr(session._lp, "_lightpanda_fallback_reason", lambda *a: None)
+ info = {"session_name": "h_abc", "cdp_url": None, "features": {"local": True}}
+
+ monkeypatch.setattr(browser, "running_instance_cdp_port", lambda d, **kw: 41234)
+ session._run_browser_command_unfenced("t", "open", ["https://x"], 10, None, "agent-browser", info)
+ assert argvs[-1][:5] == ["agent-browser", "--session", "h_abc", "--cdp", "41234"]
+
+ monkeypatch.setattr(browser, "running_instance_cdp_port", lambda d, **kw: None)
+ session._run_browser_command_unfenced("t", "open", ["https://x"], 10, None, "agent-browser", info)
+ assert "--cdp" not in argvs[-1] and argvs[-1][:3] == ["agent-browser", "--session", "h_abc"]
diff --git a/tests/tools/test_bot_desktop_browser_fence.py b/tests/tools/test_bot_desktop_browser_fence.py
new file mode 100644
index 0000000000000..743deec2de8be
--- /dev/null
+++ b/tests/tools/test_bot_desktop_browser_fence.py
@@ -0,0 +1,78 @@
+"""The bot's browser tools obey the screen lease: while a human holds the shared browser, nothing is
+dispatched, and a command whose run crossed a takeover loses its result."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from tools.bot_desktop import lease, runtime
+
+
+@pytest.fixture(autouse=True)
+def _fresh(monkeypatch):
+ lease._reset_for_tests()
+ monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":37"})
+ yield
+ lease._reset_for_tests()
+
+
+def _wire(monkeypatch, commands):
+ from tools import browser_tool as browser
+ from tools import browser_tool_session as session
+
+ monkeypatch.delenv("AGENT_BROWSER_PROFILE", raising=False)
+ monkeypatch.setattr(browser, "_is_camofox_mode", lambda: False)
+ monkeypatch.setattr(browser, "_blocked_private_page_action", lambda *a: None)
+ monkeypatch.setattr(session, "_browser_command_preflight", lambda: {"browser_cmd": "agent-browser"})
+ monkeypatch.setattr(session, "_get_session_info", lambda *a: {"session_name": "review", "cdp_url": None, "features": {"local": True}})
+ monkeypatch.setattr(session._cloud, "_get_browser_engine", lambda: "chrome")
+ monkeypatch.setattr(session._cloud, "_is_headed_mode", lambda: True)
+
+ def spawn(*args):
+ commands.append(args[2])
+ return {"success": True, "data": {"secret": "WHAT-THE-HUMAN-TYPED"}}
+
+ monkeypatch.setattr(session, "_spawn_and_collect", spawn)
+ return browser, session
+
+
+def test_browser_click_is_fenced_while_human_controls_shared_browser(monkeypatch):
+ commands: list = []
+ browser, _ = _wire(monkeypatch, commands)
+ lease.acquire("human-viewer")
+ result = json.loads(browser.browser_click("e1", task_id="review"))
+ assert commands == [], f"human holds the lease, yet a browser command was dispatched: {commands}"
+ assert result.get("code") == "human_has_control"
+
+
+def test_browser_result_crossing_a_takeover_is_discarded(monkeypatch):
+ commands: list = []
+ browser, session = _wire(monkeypatch, commands)
+
+ def spawn_then_takeover(*args):
+ lease.acquire("human-viewer")
+ lease.release("human-viewer") # a full cycle, control is back — the frame is still theirs
+ return {"success": True, "data": {"secret": "WHAT-THE-HUMAN-TYPED"}}
+
+ monkeypatch.setattr(session, "_spawn_and_collect", spawn_then_takeover)
+ result = browser.browser_click("e1", task_id="review")
+ assert "WHAT-THE-HUMAN-TYPED" not in result
+
+
+def test_real_profile_local_browser_is_fenced_by_provenance_even_without_a_live_display(monkeypatch):
+ """A real-profile session attaches over a loopback cdp_url but is launched with the Bot Desktop
+ DISPLAY, so it IS the human's browser: the fence keys on the ``local`` feature, not on the
+ transport. And a stranded human lease with the screen already down must still fence (computer_use
+ does), not silently unfence the browser."""
+ commands: list = []
+ browser, session = _wire(monkeypatch, commands)
+ monkeypatch.setattr(session, "_get_session_info", lambda *a: {
+ "session_name": "rp_1", "cdp_url": "ws://127.0.0.1:9222/devtools/browser/x",
+ "features": {"local": True, "real_profile": True}})
+ monkeypatch.setattr(runtime, "published_env", lambda: {})
+ lease.acquire("human-viewer")
+ result = json.loads(browser.browser_click("e1", task_id="review"))
+ assert commands == [], f"human holds the lease, yet a real-profile browser command was dispatched: {commands}"
+ assert result.get("code") == "human_has_control"
diff --git a/tests/tools/test_bot_desktop_install.py b/tests/tools/test_bot_desktop_install.py
new file mode 100644
index 0000000000000..a3e5392a66a28
--- /dev/null
+++ b/tests/tools/test_bot_desktop_install.py
@@ -0,0 +1,99 @@
+"""Bot Desktop package install: sudo hand-off and single-flight invariants."""
+
+from __future__ import annotations
+
+import threading
+
+import pytest
+
+from tools.bot_desktop import install, runtime
+
+
+@pytest.fixture(autouse=True)
+def _isolated_host(tmp_path, monkeypatch):
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+ monkeypatch.setattr(install, "_sudo_nopasswd", lambda: False)
+ monkeypatch.setattr(runtime, "is_supported_host", lambda: True)
+ monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y tigervnc-standalone-server")
+ yield
+ install._running.clear()
+
+
+def test_empty_password_cancels_without_spawning(monkeypatch):
+ monkeypatch.setattr(install.subprocess, "Popen", lambda *a, **k: pytest.fail("package manager spawned"))
+ lines: list[str] = []
+ code = install.install_packages(ask_password=lambda: "", on_line=lines.append)
+ assert code == -1
+ assert any("cancelled" in line for line in lines)
+
+
+def test_second_install_for_same_profile_is_refused(monkeypatch):
+ gate = threading.Event()
+ entered = threading.Event()
+
+ def slow_run(cmd, *, ask_password, on_line, timeout_seconds):
+ entered.set()
+ gate.wait(5)
+ return 0
+
+ monkeypatch.setattr(install, "_run", slow_run)
+ worker = threading.Thread(target=install.install_packages, kwargs={"ask_password": lambda: "pw", "on_line": lambda _l: None})
+ worker.start()
+ assert entered.wait(5)
+ with pytest.raises(install.InstallBusy):
+ install.install_packages(ask_password=lambda: "pw", on_line=lambda _l: None)
+ gate.set()
+ worker.join(5)
+ install.assert_not_running() # slot released once the run finishes
+
+
+def test_claim_is_atomic_and_refuses_a_second_claim():
+ """The gateway claims BEFORE spawning its worker; a second Install click must fail at claim time, not
+ pass a read-only check and race the worker for the slot."""
+ key = install.claim()
+ with pytest.raises(install.InstallBusy):
+ install.claim()
+ with pytest.raises(install.InstallBusy):
+ install.install_packages(ask_password=lambda: "pw", on_line=lambda _l: None)
+ install.release(key)
+ install.claim() # free again
+ install.release(key)
+
+
+@pytest.mark.linux_only
+def test_timeout_kills_the_package_managers_whole_process_group(monkeypatch):
+ """sudo forks the package manager into the same (new) session; killing sudo alone leaves apt/dnf
+ holding the dpkg lock as root. The timeout must take the group."""
+ import subprocess
+ import time
+
+ monkeypatch.setattr(install, "_sudo_nopasswd", lambda: True)
+ # stand-in for `sudo apt-get ...`: a parent that spawns a child and waits, both in the new session
+ fake = ["sudo"]
+ real_popen = subprocess.Popen
+
+ def popen(argv, **kw):
+ if argv[:1] != fake:
+ return real_popen(argv, **kw)
+ return real_popen(["bash", "-c", "sleep 30 >/dev/null 2>&1 & echo child $!; wait"], **kw)
+
+ monkeypatch.setattr(install.subprocess, "Popen", popen)
+ lines: list[str] = []
+ code = install._run("sudo apt-get install -y x", ask_password=lambda: "", on_line=lines.append, timeout_seconds=0.5)
+ assert code != 0
+ child = next(int(line.split()[1]) for line in lines if line.startswith("child "))
+ from pathlib import Path
+
+ def gone() -> bool: # /proc-based: a reparented orphan sits outside our subtree, where os.kill(pid, 0) is guarded
+ try:
+ return "Z" in (Path(f"/proc/{child}/stat").read_text().rsplit(")", 1)[1].split() or ["Z"])[0]
+ except OSError:
+ return True
+
+ for _ in range(50): # the child must die with the group, not linger reparented to init
+ if gone():
+ break
+ time.sleep(0.1)
+ else:
+ subprocess.run(["kill", "-9", str(child)], check=False)
+ pytest.fail("grandchild survived the install timeout")
diff --git a/tests/tools/test_bot_desktop_launcher_seed.py b/tests/tools/test_bot_desktop_launcher_seed.py
new file mode 100644
index 0000000000000..261d9d00558dc
--- /dev/null
+++ b/tests/tools/test_bot_desktop_launcher_seed.py
@@ -0,0 +1,64 @@
+"""Bot Desktop launcher seeds: the dock only points at programs that exist, the look is applied."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import pytest
+
+LAUNCHER = Path(__file__).resolve().parents[2] / "tools" / "bot_desktop" / "launcher.sh"
+pytestmark = pytest.mark.linux_only
+
+
+def _seed(tmp_path: Path, fake_bins: list[str], browser_exec: str = "") -> Path:
+ bindir = tmp_path / "bin"
+ bindir.mkdir()
+ for name in fake_bins:
+ exe = bindir / name
+ exe.write_text("#!/bin/sh\n", encoding="utf-8")
+ exe.chmod(0o755)
+ # The script's own tooling (mkdir, sed, cat, awk...) symlinked in, so PATH need not contain the
+ # host's /usr/bin where a real chrome/thunar would leak into the dock under test.
+ for tool in ("mkdir", "sed", "cat", "printf", "dirname", "bash", "sh", "rm", "ln", "touch", "chmod", "xauth", "od", "tr", "awk"):
+ real = shutil.which(tool)
+ if real and not (bindir / tool).exists():
+ (bindir / tool).symlink_to(real)
+ cfg = tmp_path / "xdg"
+ env = {
+ "PATH": str(bindir),
+ "HOME": str(tmp_path),
+ "HERMES_BD_PROFILE": "t", "HERMES_BD_DISPLAY_NUM": "99",
+ "HERMES_BD_SOCKET": str(tmp_path / "rfb.sock"), "HERMES_BD_XAUTH": str(tmp_path / "Xauthority"),
+ "HERMES_BD_ENV_FILE": str(tmp_path / "env"), "HERMES_BD_CONFIG_HOME": str(cfg),
+ "HERMES_BD_SEED_ONLY": "1",
+ **({"HERMES_BD_BROWSER_EXEC": browser_exec} if browser_exec else {}),
+ }
+ subprocess.run(["bash", str(LAUNCHER)], env=env, check=True, stdin=subprocess.DEVNULL, capture_output=True, timeout=30)
+ return cfg
+
+
+def test_dock_lists_only_programs_present_on_path(tmp_path):
+ chrome = tmp_path / "bin" / "chrome" # the browser is the one runtime.py resolved, never a PATH scan
+ cfg = _seed(tmp_path, ["xfce4-terminal", "chrome", "firefox"], browser_exec=f"{chrome} --user-data-dir={tmp_path}/bp")
+ panel = ET.parse(cfg / "xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml") # well-formed or this raises
+ launcher_ids = [str(p.get("name")) for p in panel.iter("property") if p.get("value") == "launcher"]
+ execs = sorted(
+ line.split("=", 1)[1]
+ for pid in launcher_ids
+ for line in (cfg / "xfce4/panel" / pid.replace("plugin-", "launcher-") / "hermes.desktop").read_text(encoding="utf-8").splitlines()
+ if line.startswith("Exec=")
+ )
+ assert execs == [f"{chrome} --user-data-dir={tmp_path}/bp", "xfce4-terminal"]
+
+
+def test_look_is_seeded_with_wallpaper_and_theme(tmp_path):
+ cfg = _seed(tmp_path, ["xfce4-terminal"])
+ desktop = (cfg / "xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml").read_text(encoding="utf-8")
+ xsettings = (cfg / "xfce4/xfconf/xfce-perchannel-xml/xsettings.xml").read_text(encoding="utf-8")
+ assert str(LAUNCHER.with_name("wallpaper.png")) in desktop
+ assert "PLACEHOLDER" not in desktop + xsettings
+ assert os.path.isfile(LAUNCHER.with_name("wallpaper.png"))
diff --git a/tests/tools/test_bot_desktop_launcher_xvnc.py b/tests/tools/test_bot_desktop_launcher_xvnc.py
new file mode 100644
index 0000000000000..c2861796c2491
--- /dev/null
+++ b/tests/tools/test_bot_desktop_launcher_xvnc.py
@@ -0,0 +1,41 @@
+"""The Bot Desktop's Xvnc is started with the RFB options the lease design relies on."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+LAUNCHER = Path(__file__).resolve().parents[2] / "tools" / "bot_desktop" / "launcher.sh"
+pytestmark = pytest.mark.linux_only
+
+
+def test_xvnc_never_sends_the_holders_clipboard_to_watchers(tmp_path):
+ """Whoever holds control may paste INTO the screen (AcceptCutText), but the screen's clipboard must
+ not be pushed to every connected viewer (SendCutText off): watchers are not the holder."""
+ bindir = tmp_path / "bin"
+ bindir.mkdir()
+ argv_log = tmp_path / "xvnc-argv"
+ (bindir / "Xvnc").write_text(f'#!/bin/sh\nprintf "%s\\n" "$@" > "{argv_log}"\nexec sleep 3\n', encoding="utf-8")
+ for stub in ("xdpyinfo", "setxkbmap", "xsetroot", "xset", "dbus-run-session"):
+ (bindir / stub).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+ for exe in bindir.iterdir():
+ exe.chmod(0o755)
+ for tool in ("mkdir", "sed", "cat", "printf", "dirname", "bash", "sh", "rm", "ln", "touch", "chmod", "xauth", "od", "tr", "awk", "seq", "sleep", "kill"):
+ real = shutil.which(tool)
+ if real and not (bindir / tool).exists():
+ (bindir / tool).symlink_to(real)
+ env = {
+ "PATH": str(bindir), "HOME": str(tmp_path),
+ "HERMES_BD_PROFILE": "t", "HERMES_BD_DISPLAY_NUM": "99",
+ "HERMES_BD_SOCKET": str(tmp_path / "rfb.sock"), "HERMES_BD_XAUTH": str(tmp_path / "Xauthority"),
+ "HERMES_BD_ENV_FILE": str(tmp_path / "env"), "HERMES_BD_CONFIG_HOME": str(tmp_path / "xdg"),
+ }
+ subprocess.run(["bash", str(LAUNCHER)], env=env, check=True, stdin=subprocess.DEVNULL, capture_output=True, timeout=30)
+ argv = argv_log.read_text(encoding="utf-8").split("\n")
+ assert "-SendCutText=0" in argv, argv
+ assert not any(a.startswith("-AcceptCutText") for a in argv), "paste into the screen must keep working"
+ assert not os.path.exists(tmp_path / "rfb.sock") # stub never bound it; nothing leaked
diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py
new file mode 100644
index 0000000000000..50f948f46ad57
--- /dev/null
+++ b/tests/tools/test_bot_desktop_lease.py
@@ -0,0 +1,152 @@
+"""Bot Desktop invariants: the byte-level RFB input gate follows the lease, and computer_use refuses
+every action (capture included) while a human holds the screen."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from tools.bot_desktop import lease
+from tools.bot_desktop.rfb_filter import RfbClientFilter
+
+_HANDSHAKE = b"RFB 003.008\n" + b"\x01" + b"\x00"
+_KEY = b"\x04\x01\x00\x00\x00\x00\x00\x61" # KeyEvent 'a' down
+# QEMU Extended KeyEvent (type 255, sub 0): what noVNC sends once Xvnc advertises the pseudo-encoding.
+_QEMU_KEY = bytes([255, 0, 0, 1]) + (0x65).to_bytes(4, "big") + (0x12).to_bytes(4, "big")
+_POINTER = b"\x05\x01\x00\x10\x00\x10" # PointerEvent, button 1
+_CUT = b"\x06\x00\x00\x00\x00\x00\x00\x02hi" # ClientCutText "hi"
+_FBUR = b"\x03\x00" + b"\x00" * 8 # FramebufferUpdateRequest
+_SETENC = b"\x02\x00\x00\x02" + b"\x00\x00\x00\x07" + b"\xff\xff\xff\x21" # SetEncodings x2
+
+
+@pytest.fixture(autouse=True)
+def _fresh_lease():
+ lease._reset_for_tests()
+ yield
+ lease._reset_for_tests()
+
+
+def test_rfb_filter_forwards_input_only_from_the_lease_holder_across_arbitrary_chunking():
+ f = RfbClientFilter(lambda: lease.viewer_may_send_input("v1"))
+ head = f.feed(_HANDSHAKE)
+ assert head[-1:] == b"\x01", "ClientInit is forced shared so a viewer never kicks the agent's watcher"
+
+ # Agent holds: read-only messages pass, input is dropped, even when split byte by byte.
+ stream = _KEY + _FBUR + _POINTER + _SETENC + _CUT + _QEMU_KEY
+ out = b"".join(f.feed(stream[i:i + 1]) for i in range(len(stream)))
+ assert out == _FBUR + _SETENC
+
+ lease.acquire("v1")
+ assert f.feed(_KEY + _POINTER + _QEMU_KEY) == _KEY + _POINTER + _QEMU_KEY
+
+ lease.acquire("v2") # last writer wins: v1 is evicted from input on the very next message
+ assert f.feed(_KEY) == b""
+ assert lease.release("v1").holder == lease.HUMAN, "a stale viewer's release must not yank control from v2"
+ assert lease.release("v2").holder == lease.AGENT
+
+
+def test_computer_use_refuses_every_action_while_a_human_holds_the_screen(monkeypatch):
+ from tools.computer_use import tool
+
+ calls = []
+ monkeypatch.setattr(tool, "_get_backend", lambda session_id="": calls.append(session_id) or object())
+ lease.acquire("human")
+ for action in ("capture", "click", "type", "list_windows"):
+ res = json.loads(tool.handle_computer_use({"action": action, "text": "pw"}))
+ assert res["code"] == "human_has_control", action
+ assert calls == [], "the driver is never touched while the human may be typing a credential"
+
+ # Handoff round trip: the agent asks, the human takes over and hands back, the agent is unblocked.
+ asked = json.loads(tool.handle_computer_use({"action": "request_handoff", "reason": "log in"}))
+ assert asked["ok"] and asked["state"]["pending_handoff"] == "log in"
+ lease.acquire("human", reason="log in")
+ assert lease.get().pending_handoff is None
+ lease.release("human")
+ done = json.loads(tool.handle_computer_use({"action": "wait_for_human", "seconds": 1}))
+ assert done["ok"] and done["state"]["holder"] == lease.AGENT
+
+
+def test_lease_authority_is_shared_across_processes(tmp_path):
+ """The gateway that streams the screen and the process running the agent are different processes;
+ a human takeover in one must refuse actions in the other."""
+ import os
+ import subprocess
+ import sys
+
+ lease.acquire("desktop-viewer")
+ probe = ("import sys; sys.path.insert(0, %r)\n"
+ "from tools.bot_desktop import lease\n"
+ "try:\n lease.assert_agent_may_act(); print('AGENT')\n"
+ "except lease.HumanHasControl:\n print('HUMAN')\n"
+ "lease.release('desktop-viewer')\n") % os.getcwd()
+ out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, encoding="utf-8", timeout=30,
+ stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": os.environ["HERMES_HOME"]})
+ assert out.stdout.strip() == "HUMAN", out.stderr
+ assert lease.get().holder == lease.AGENT, "the other process's release is visible here"
+
+
+def test_takeover_during_an_admitted_action_discards_its_result(monkeypatch):
+ """Approval / backend start-up can take seconds; a human who takes over meanwhile must not have
+ their keystrokes captured by an action admitted before they did."""
+ from tools.computer_use import tool
+
+ monkeypatch.setattr(tool, "_get_backend", lambda session_id="": object())
+
+ def _dispatch_then_takeover(backend, action, args, **_):
+ lease.acquire("human") # a whole take-over / hand-back cycle inside the driver call:
+ lease.release("human") # control is back, but the frame is still the human's turn
+ return json.dumps({"ok": True, "action": action, "png_b64": "SECRET"})
+
+ monkeypatch.setattr(tool, "_dispatch", _dispatch_then_takeover)
+ res = json.loads(tool.handle_computer_use({"action": "capture"}))
+ assert res["code"] == "human_has_control" and "SECRET" not in json.dumps(res)
+
+
+def test_unreadable_lease_file_fails_closed_and_takeover_keeps_the_agents_reason(tmp_path):
+ """Missing file = fresh profile (agent). A file that exists but cannot be parsed must not read as
+ "agent holds": a torn write must never let the agent act on a human's screen. Taking over after a
+ request keeps the agent's reason so the human still sees WHY while they act."""
+ from hermes_constants import hermes_home_key
+
+ home = str(tmp_path)
+ assert lease.get(profile_key=home).holder == lease.AGENT
+ path = lease._path(home)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text("{ torn", encoding="utf-8")
+ assert lease.get(profile_key=home).holder == lease.HUMAN
+ lease.release(profile_key=home) # a successful write repairs it
+ assert lease.get(profile_key=home).holder == lease.AGENT
+
+ # Valid JSON of the wrong shape is just as untrustworthy as torn JSON: never read it as "agent holds".
+ for wrong_shape in ("[]", "null", "5", "{}", '{"holder": "root"}'):
+ path.write_text(wrong_shape, encoding="utf-8")
+ assert lease.get(profile_key=home).holder == lease.HUMAN, wrong_shape
+ lease.release(profile_key=home)
+ assert lease.get(profile_key=home).holder == lease.AGENT
+
+ lease.request_handoff("log in to the bank, 2FA on your phone", profile_key=home)
+ held = lease.acquire("desk-1", profile_key=home)
+ assert held.pending_handoff is None and held.reason == "log in to the bank, 2FA on your phone"
+ assert hermes_home_key(home) # sanity: the key derivation used by the bridge is available
+
+
+def test_lease_works_without_fcntl(tmp_path):
+ """Windows and fcntl-less hosts: ``computer_use`` imports the lease (via handoff) on EVERY call, so a
+ module-level fcntl dependency turns every desktop action into ModuleNotFoundError there. The file
+ semantics must still work; only the cross-process lock degrades. Subprocess so the module cache is clean."""
+ import os
+ import subprocess
+ import sys
+
+ probe = ("import sys; sys.modules['fcntl'] = None; sys.path.insert(0, %r)\n"
+ "from tools.bot_desktop import lease\n"
+ "import tools.computer_use.handoff\n"
+ "assert lease.get().holder == lease.AGENT\n"
+ "assert lease.acquire('v1').holder == lease.HUMAN\n"
+ "assert lease.get().holder == lease.HUMAN\n"
+ "assert lease.release('v1').holder == lease.AGENT\n"
+ "print('OK')\n") % os.getcwd()
+ out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, encoding="utf-8", timeout=60,
+ stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": str(tmp_path)})
+ assert out.stdout.strip() == "OK", out.stderr
diff --git a/tests/tools/test_bot_desktop_rfb_filter_limits.py b/tests/tools/test_bot_desktop_rfb_filter_limits.py
new file mode 100644
index 0000000000000..018336d9f6406
--- /dev/null
+++ b/tests/tools/test_bot_desktop_rfb_filter_limits.py
@@ -0,0 +1,24 @@
+"""A client-declared ClientCutText length is bounded at the header: the bridge must not buffer up to
+2 GiB for a viewer that holds a ticket but no lease."""
+
+import pytest
+
+from tools.bot_desktop.rfb_filter import _MAX_CUT_TEXT, RfbClientFilter
+
+_HANDSHAKE = b"RFB 003.008\n\x01\x01"
+
+
+def clipboard_header(length):
+ return b"\x06\x00\x00\x00" + length.to_bytes(4, "big", signed=True)
+
+
+@pytest.mark.parametrize("length", [_MAX_CUT_TEXT + 1, -_MAX_CUT_TEXT - 1, 2**31 - 1, -(2**31)])
+@pytest.mark.parametrize("holder", [False, True])
+def test_oversized_clipboard_is_rejected_at_header_without_waiting_for_payload(length, holder):
+ parser = RfbClientFilter(lambda: holder)
+ parser.feed(_HANDSHAKE)
+ header = clipboard_header(length)
+ for byte in header[:-1]:
+ assert parser.feed(bytes([byte])) == b""
+ with pytest.raises(ValueError, match="clipboard"):
+ parser.feed(header[-1:])
diff --git a/tests/tools/test_bot_desktop_runtime.py b/tests/tools/test_bot_desktop_runtime.py
new file mode 100644
index 0000000000000..fd89a14d6de8f
--- /dev/null
+++ b/tests/tools/test_bot_desktop_runtime.py
@@ -0,0 +1,146 @@
+"""Bot Desktop runtime: thumbnail and display-number allocation invariants."""
+
+from __future__ import annotations
+
+import contextlib
+import sys
+from pathlib import Path
+
+import pytest
+
+from tools.bot_desktop import runtime, thumbnail
+
+
+@pytest.mark.parametrize("pm", sorted(runtime.PACKAGES))
+def test_every_required_binary_maps_to_an_installed_package(pm):
+ """Each binary the launcher execs must come from a package the distro list actually installs; dnf5
+ refuses the whole transaction on one retired name, so the map is the contract, not the list."""
+ mapping = runtime.BINARY_PACKAGES[pm]
+ assert set(mapping) == set(runtime.REQUIRED_BINARIES)
+ assert set(mapping.values()) <= set(runtime.PACKAGES[pm])
+ assert not {"xorg-x11-server-utils", "xorg-x11-utils"} & set(runtime.PACKAGES["dnf"]), "retired on Fedora"
+
+
+def test_no_running_screen_returns_none_without_grabbing(monkeypatch):
+ monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":99"})
+ monkeypatch.setattr(runtime, "_launcher_pid", lambda: None)
+ monkeypatch.setitem(sys.modules, "PIL.ImageGrab", None) # an import would now fail loudly
+ assert thumbnail.thumbnail_data_url() is None
+
+
+def test_recycled_pid_is_not_our_launcher(tmp_path, monkeypatch):
+ """launcher.pid names pid + create_time; a live pid born at another time is a stranger (recycled pid)
+ and must read as not running, or stop() would killpg an unrelated session. Legacy single-number
+ files and absurd digit strings are also not running."""
+ import os
+
+ monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path)
+ pidfile = tmp_path / "launcher.pid"
+ pidfile.write_text(f"{os.getpid()} 12345.0", encoding="utf-8") # alive, wrong birth
+ assert runtime._launcher_pid() is None
+ pidfile.write_text(str(os.getpid()), encoding="utf-8") # pre-identity format
+ assert runtime._launcher_pid() is None
+ pidfile.write_text("9" * 40 + " 1.0", encoding="utf-8")
+ assert runtime._launcher_pid() is None
+ pidfile.write_text(f"{os.getpid()} {runtime._create_time(os.getpid())}", encoding="utf-8")
+ assert runtime._launcher_pid() == os.getpid()
+
+
+def test_recorded_display_held_by_a_live_server_is_not_reused(tmp_path, monkeypatch):
+ """After profile A stops, B may take A's number; A restarting must pick another rather than
+ unlink B's socket and lock."""
+ import os
+
+ monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path)
+ (tmp_path / "display").write_text("37", encoding="utf-8")
+ live = {37: os.getpid()} # :37 is owned by a running server (this very process stands in for it)
+ monkeypatch.setattr(runtime, "_display_in_use", lambda num: num in live)
+ monkeypatch.setattr(runtime, "_ALLOC_LOCK", tmp_path / "alloc.lock")
+ assert runtime._allocate_display() != 37
+ live.clear()
+ assert runtime._allocate_display() == 37, "a free recorded number is reclaimed"
+
+
+
+_FAKE_LAUNCHER = """#!/usr/bin/env bash
+# Stands in for launcher.sh + Xvnc: the X lock appears only after a delay (the TOCTOU window), then the
+# env file + socket are published; stays alive until killed like the real supervisor.
+: > "$HERMES_BD_XLOCK_DIR/spawned.$$"
+sleep 0.4
+echo $$ > "$HERMES_BD_XLOCK_DIR/.X${HERMES_BD_DISPLAY_NUM}-lock"
+: > "$HERMES_BD_SOCKET"
+printf 'DISPLAY=:%s\\n' "$HERMES_BD_DISPLAY_NUM" > "$HERMES_BD_ENV_FILE"
+sleep 30
+"""
+
+# One start() per process: state_dir() is HERMES_HOME-scoped and process-global, so two profiles need two
+# interpreters — which is also how two gateway profiles race on a real host.
+_DRIVER = """
+import json, os, sys
+from pathlib import Path
+sys.path.insert(0, {repo!r})
+from tools.bot_desktop import runtime
+scratch = Path({scratch!r})
+runtime._LAUNCHER = scratch / "launcher.sh"
+runtime._X_LOCK_DIR = scratch / "xlocks"
+runtime._ALLOC_LOCK = scratch / "alloc.lock"
+runtime.missing_binaries = lambda: []
+runtime.geometry = lambda: "800x600"
+os.environ["HERMES_BD_XLOCK_DIR"] = str(scratch / "xlocks")
+try:
+ st = runtime.start(wait_seconds=10)
+ print(json.dumps({{"display": st.display, "pid": st.pid}}), flush=True)
+except Exception as exc:
+ print(json.dumps({{"error": str(exc)}}), flush=True)
+sys.stdin.readline() # the test releases us once every driver has reported; we own the launcher, we stop it
+runtime.stop()
+"""
+
+
+@pytest.fixture
+def start_in_fresh_process(tmp_path):
+ import os
+ import subprocess
+
+ (tmp_path / "launcher.sh").write_text(_FAKE_LAUNCHER, encoding="utf-8")
+ (tmp_path / "xlocks").mkdir()
+ repo = str(Path(__file__).resolve().parents[2])
+ procs: list[subprocess.Popen] = []
+
+ def launch(home: Path) -> subprocess.Popen:
+ env = {**os.environ, "HERMES_HOME": str(home)}
+ proc = subprocess.Popen([sys.executable, "-c", _DRIVER.format(repo=repo, scratch=str(tmp_path))],
+ env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
+ procs.append(proc)
+ return proc
+
+ yield launch
+ for proc in procs:
+ with contextlib.suppress(OSError):
+ proc.communicate("go\n", timeout=20)
+ proc.kill()
+
+
+def _collect(procs):
+ import json
+ out = [json.loads(p.stdout.readline()) for p in procs] # every driver holds its launcher until released
+ assert all("error" not in o for o in out), out
+ return out
+
+
+@pytest.mark.linux_only
+def test_concurrent_cold_starts_of_two_profiles_get_distinct_displays(tmp_path, start_in_fresh_process):
+ """The allocation lock must outlive the pick: Xvnc writes /tmp/.X-lock well after start() chose n, so
+ a second profile starting in that window used to pick the same n (and its launcher's stale-lock cleanup
+ could then unlink the winner's socket)."""
+ out = _collect([start_in_fresh_process(tmp_path / "a"), start_in_fresh_process(tmp_path / "b")])
+ assert len({o["display"] for o in out}) == 2, out
+
+
+@pytest.mark.linux_only
+def test_concurrent_starts_of_one_profile_spawn_one_launcher(tmp_path, start_in_fresh_process):
+ """Two start() calls for one profile spawn ONE launcher; the second used to spawn its own, overwrite
+ launcher.pid and orphan the first (both callers then reported the last-written pid)."""
+ out = _collect([start_in_fresh_process(tmp_path / "a"), start_in_fresh_process(tmp_path / "a")])
+ assert len({o["pid"] for o in out}) == 1, out
+ assert len(list((tmp_path / "xlocks").glob("spawned.*"))) == 1
diff --git a/tests/tools/test_bot_desktop_thumbnail.py b/tests/tools/test_bot_desktop_thumbnail.py
new file mode 100644
index 0000000000000..73f742c69afa2
--- /dev/null
+++ b/tests/tools/test_bot_desktop_thumbnail.py
@@ -0,0 +1,41 @@
+"""``thumbnail_data_url`` swaps the process-wide XAUTHORITY around the grab; two profiles grabbed on
+worker threads at once must each see their own cookie file, not the other's."""
+
+from __future__ import annotations
+
+import os
+import threading
+
+from PIL import Image, ImageGrab
+
+from tools.bot_desktop import runtime, thumbnail
+
+
+def test_concurrent_grabs_each_see_their_own_xauthority(monkeypatch):
+ envs = {"a": {"DISPLAY": ":91", "XAUTHORITY": "/tmp/xauth-a"},
+ "b": {"DISPLAY": ":92", "XAUTHORITY": "/tmp/xauth-b"}}
+ local = threading.local()
+ monkeypatch.setattr(runtime, "published_env", lambda: envs[local.profile])
+ monkeypatch.setattr(runtime, "_launcher_pid", lambda: 4242)
+ monkeypatch.delenv("XAUTHORITY", raising=False)
+ seen = {}
+ start = threading.Barrier(2)
+
+ def fake_grab(xdisplay=None):
+ seen[xdisplay] = os.environ.get("XAUTHORITY")
+ threading.Event().wait(0.05) # hold the env long enough for the other thread to collide
+ return Image.new("RGB", (8, 8))
+ monkeypatch.setattr(ImageGrab, "grab", fake_grab)
+
+ def worker(profile):
+ local.profile = profile
+ start.wait()
+ thumbnail.thumbnail_data_url()
+
+ threads = [threading.Thread(target=worker, args=(p,)) for p in envs]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(5)
+ assert seen == {":91": "/tmp/xauth-a", ":92": "/tmp/xauth-b"}
+ assert "XAUTHORITY" not in os.environ
diff --git a/tests/tools/test_computer_use_backend_rebind.py b/tests/tools/test_computer_use_backend_rebind.py
new file mode 100644
index 0000000000000..c2ca5b99071a1
--- /dev/null
+++ b/tests/tools/test_computer_use_backend_rebind.py
@@ -0,0 +1,13 @@
+"""A cached computer_use backend is bound to the display it spawned on; the identity helpers notice a change."""
+
+from __future__ import annotations
+
+from tools.computer_use import cua_backend
+
+
+def test_backend_display_identity_tracks_the_display_a_spawn_would_get():
+ before = cua_backend.desktop_identity({"HOME": "/x"}) # no screen yet
+ after = cua_backend.desktop_identity({"HOME": "/x", "DISPLAY": ":37"}) # Bot Desktop came up
+ assert before == "" and after == ":37"
+ assert cua_backend.backend_display_stale(before, after)
+ assert not cua_backend.backend_display_stale(after, cua_backend.desktop_identity({"DISPLAY": ":37"}))
diff --git a/tests/tools/test_computer_use_capture_fence.py b/tests/tools/test_computer_use_capture_fence.py
new file mode 100644
index 0000000000000..65736e5ca12ef
--- /dev/null
+++ b/tests/tools/test_computer_use_capture_fence.py
@@ -0,0 +1,56 @@
+"""A frame captured across a human takeover never leaves the process: the lease fence runs as soon as the
+backend hands the frame back, before it is persisted to the media cache, spilled, or routed to aux vision."""
+
+from __future__ import annotations
+
+import base64
+import json
+
+import pytest
+
+from tools.bot_desktop import lease
+from tools.computer_use.backend import ActionResult, CaptureResult
+
+
+@pytest.fixture(autouse=True)
+def _fresh_lease():
+ lease._reset_for_tests()
+ yield
+ lease._reset_for_tests()
+
+
+class _TakeoverBackend:
+ """Driver whose capture returns while a take-over / hand-back cycle happened underneath it."""
+ _last_target = None
+ _last_app = None
+
+ def capture(self, **kw):
+ lease.acquire("human")
+ lease.release("human")
+ return CaptureResult(mode="som", width=64, height=64, png_b64=base64.b64encode(b"SECRETPNG").decode(), elements=[],
+ app="Bank", window_title="login")
+
+ def click(self, **kw):
+ return ActionResult(ok=True, action="click")
+
+
+def _spy_sinks(monkeypatch, tool):
+ leaked: list = []
+ monkeypatch.setattr(tool, "_persist_capture_image", lambda cap: leaked.append(("persist", cap)))
+ monkeypatch.setattr(tool, "_spill_elements_to_file", lambda cap: leaked.append(("spill", cap)))
+ monkeypatch.setattr(tool, "_should_route_through_aux_vision", lambda: leaked.append(("aux-decide",)) or True)
+ monkeypatch.setattr(tool, "_route_capture_through_aux_vision", lambda cap, summary, **kw: leaked.append(("aux", cap)))
+ return leaked
+
+
+@pytest.mark.parametrize("args", [{"action": "capture"}, {"action": "click", "coordinate": [1, 1], "capture_after": True}])
+def test_frame_captured_across_a_takeover_is_dropped_before_any_sink(monkeypatch, args):
+ from tools.computer_use import tool
+
+ monkeypatch.setattr(tool, "_get_backend", lambda session_id="": _TakeoverBackend())
+ monkeypatch.setattr(tool, "_request_approval", lambda *a, **k: None)
+ leaked = _spy_sinks(monkeypatch, tool)
+ res = json.loads(tool.handle_computer_use(args))
+ assert res["code"] == "human_has_control", res
+ assert leaked == [], f"the human's frame reached a sink: {leaked}"
+ assert "SECRETPNG" not in json.dumps(res)
diff --git a/tests/tools/test_computer_use_handoff_wait.py b/tests/tools/test_computer_use_handoff_wait.py
new file mode 100644
index 0000000000000..29811d8580540
--- /dev/null
+++ b/tests/tools/test_computer_use_handoff_wait.py
@@ -0,0 +1,38 @@
+"""wait_for_human answers an unanswered handoff early instead of blocking the whole timeout, and still
+waits the full timeout once a human actually holds the screen."""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+
+import pytest
+
+from tools.bot_desktop import lease
+from tools.computer_use.handoff import handle_handoff
+
+
+@pytest.fixture(autouse=True)
+def _fresh_lease():
+ lease._reset_for_tests()
+ yield
+ lease._reset_for_tests()
+
+
+def test_wait_for_human_returns_no_takeover_when_nobody_answers_but_waits_out_a_real_takeover():
+ handle_handoff("request_handoff", {"reason": "log in"})
+ t0 = time.monotonic()
+ res = json.loads(handle_handoff("wait_for_human", {"seconds": 30, "grace": 0.2}))
+ assert res["code"] == "no_takeover" and res["state"]["pending_handoff"] == "log in"
+ assert time.monotonic() - t0 < 10, "an unanswered request must not run the full timeout"
+
+ # A human takes over inside the grace window and hands back later: the wait outlives the grace.
+ def _take_then_release():
+ time.sleep(0.1)
+ lease.acquire("viewer-1")
+ time.sleep(0.6)
+ lease.release("viewer-1")
+ threading.Thread(target=_take_then_release, daemon=True).start()
+ res = json.loads(handle_handoff("wait_for_human", {"seconds": 30, "grace": 0.3}))
+ assert res["ok"] and res["state"]["holder"] == lease.AGENT
diff --git a/tests/tools/test_computer_use_native_platform.py b/tests/tools/test_computer_use_native_platform.py
new file mode 100644
index 0000000000000..4a018eb39fb99
--- /dev/null
+++ b/tests/tools/test_computer_use_native_platform.py
@@ -0,0 +1,38 @@
+"""Exercise existing computer-use actions on the actual Windows/macOS CI hosts."""
+
+import json
+import sys
+
+import pytest
+
+from tools.computer_use import tool
+
+
+@pytest.fixture
+def backend(monkeypatch):
+ tool.reset_backend_for_tests()
+ monkeypatch.setenv("HERMES_COMPUTER_USE_BACKEND", "noop")
+ value = tool._get_backend()
+ yield value
+ tool.reset_backend_for_tests()
+
+
+@pytest.mark.parametrize("host_platform", [
+ pytest.param("win32", marks=pytest.mark.windows_only),
+ pytest.param("darwin", marks=pytest.mark.macos_only),
+])
+@pytest.mark.parametrize("args, expected_call", [
+ ({"action": "capture", "mode": "ax"}, "capture"),
+ ({"action": "click", "coordinate": [10, 10]}, "click"),
+ ({"action": "type", "text": "test input"}, "type"),
+ ({"action": "list_windows"}, "list_windows"),
+])
+def test_native_computer_use_dispatches_to_backend(backend, args, expected_call, host_platform):
+ import tools.computer_use_tool # noqa: F401 - register the real tool handler
+ from tools.registry import registry
+
+ assert sys.platform == host_platform
+ result = registry.dispatch("computer_use", args)
+
+ assert "error" not in json.loads(result)
+ assert [name for name, _ in backend.calls] == [expected_call]
diff --git a/tests/tui_gateway/test_display_methods.py b/tests/tui_gateway/test_display_methods.py
new file mode 100644
index 0000000000000..4998946b47fa0
--- /dev/null
+++ b/tests/tui_gateway/test_display_methods.py
@@ -0,0 +1,125 @@
+"""display.install runs its worker inside the caller's profile scope; display.observe mints the viewer identity."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+
+import pytest
+
+from hermes_cli.dashboard_auth import ws_tickets
+
+
+def test_install_worker_keeps_the_requested_profile_scope(tmp_path, monkeypatch):
+ from hermes_constants import get_hermes_home
+ from tools.bot_desktop import install, runtime
+ import tui_gateway.server as server
+
+ named = tmp_path / "profiles" / "named"
+ named.mkdir(parents=True)
+ monkeypatch.setattr(server, "_profile_home", lambda name: str(named) if name == "named" else None)
+ monkeypatch.setattr(runtime, "is_supported_host", lambda: True)
+ monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y x")
+ seen = {}
+ done = threading.Event()
+
+ def fake_install(*, ask_password, on_line, timeout_seconds=900.0, claimed=False):
+ seen["home"] = str(get_hermes_home())
+ done.set()
+ return 0
+
+ monkeypatch.setattr(install, "install_packages", fake_install)
+ monkeypatch.setattr(server, "_broadcast_global_event", lambda *a, **k: None)
+ resp = server.handle_request({"jsonrpc": "2.0", "id": 1, "method": "display.install", "params": {"profile": "named"}})
+ assert resp["result"]["started"], resp
+ assert done.wait(5)
+ assert seen["home"] == str(named)
+
+
+@pytest.fixture
+def _fresh_lease():
+ from tools.bot_desktop import lease
+ lease._reset_for_tests()
+ yield lease
+ lease._reset_for_tests()
+
+
+def _call(server, method, params):
+ return server.handle_request({"jsonrpc": "2.0", "id": 7, "method": method, "params": params})
+
+
+def test_thumbnail_is_suppressed_while_a_human_holds_the_lease(monkeypatch, _fresh_lease):
+ """The Desktop polls thumbnails on a timer; while a human drives the screen that grab would ship
+ whatever they are typing to every connected client, so it must not touch the framebuffer at all."""
+ import tui_gateway.server as server
+ from tools.bot_desktop import thumbnail
+
+ grabs = []
+ monkeypatch.setattr(thumbnail, "thumbnail_data_url", lambda: grabs.append(1) or "data:image/jpeg;base64,SECRET")
+ _fresh_lease.acquire("viewer-1")
+ result = _call(server, "display.thumbnail", {})["result"]
+ assert result["data_url"] is None and result["suppressed"] == "human_has_control"
+ assert grabs == [], "the framebuffer was grabbed while a human held the lease"
+ _fresh_lease.release("viewer-1")
+ assert _call(server, "display.thumbnail", {})["result"]["data_url"].endswith("SECRET")
+
+
+def test_release_without_viewer_id_cannot_yank_another_viewers_lease(_fresh_lease):
+ """lease.release(None) skips the holder check, so a client that lost its viewer id (or a bare RPC)
+ must be refused unless it forces; a matching viewer id and force keep working."""
+ import tui_gateway.server as server
+
+ _fresh_lease.acquire("viewer-1")
+ refused = _call(server, "display.lease.release", {})
+ assert refused["error"]["data"]["code"] == "viewer_mismatch"
+ assert _fresh_lease.get().holder == _fresh_lease.HUMAN
+ assert _call(server, "display.lease.release", {"viewer_id": "viewer-1"})["result"]["lease"]["holder"] == _fresh_lease.AGENT
+ _fresh_lease.acquire("viewer-2")
+ assert _call(server, "display.lease.release", {"force": True})["result"]["lease"]["holder"] == _fresh_lease.AGENT
+
+def _rpc(server, method, params):
+ return server.handle_request({"jsonrpc": "2.0", "id": 7, "method": method, "params": params})
+
+
+def test_observe_mints_the_viewer_id_and_status_never_discloses_the_holder(monkeypatch, tmp_path):
+ """A client cannot choose its viewer id (it would impersonate the holder and co-drive or release
+ their lease), and no snapshot or broadcast carries the raw holder id — only a hash the holder
+ itself can match."""
+ from tools.bot_desktop import lease, runtime
+ import tui_gateway.server as server
+
+ monkeypatch.setattr(runtime, "rfb_socket_path", lambda: tmp_path / "rfb.sock")
+ lease._reset_for_tests()
+ broadcasts = []
+ monkeypatch.setattr(server, "_broadcast_global_event", lambda ev, payload=None: broadcasts.append((ev, payload)))
+ try:
+ observed = _rpc(server, "display.observe", {"viewer_id": "victim"})["result"]
+ assert observed["viewer_id"] != "victim"
+ assert observed["viewer_id"] and len(observed["viewer_id"]) >= 16
+ assert ws_tickets.consume_ticket(observed["ticket"])["viewer_id"] == observed["viewer_id"]
+ holder = observed["viewer_id"]
+
+ # Only the connection that minted an id may reuse it (a reconnecting pane keeps its lease).
+ class _Peer:
+ def write(self, obj):
+ return True
+ mine, other = _Peer(), _Peer()
+ with_mine = server.dispatch({"jsonrpc": "2.0", "id": 8, "method": "display.observe", "params": {}}, mine)["result"]
+ again = server.dispatch({"jsonrpc": "2.0", "id": 9, "method": "display.observe",
+ "params": {"viewer_id": with_mine["viewer_id"]}}, mine)["result"]
+ assert again["viewer_id"] == with_mine["viewer_id"]
+ stolen = server.dispatch({"jsonrpc": "2.0", "id": 10, "method": "display.observe",
+ "params": {"viewer_id": with_mine["viewer_id"]}}, other)["result"]
+ assert stolen["viewer_id"] != with_mine["viewer_id"]
+
+ _rpc(server, "display.status", {}) # installs the broadcast listener
+ lease.acquire(holder)
+ status = _rpc(server, "display.status", {})["result"]
+ assert status["lease"]["holder"] == lease.HUMAN
+ assert holder not in json.dumps(status)
+ assert status["lease"]["viewer_hash"] == hashlib.sha256(holder.encode()).hexdigest()[:12]
+ lease_events = [p for ev, p in broadcasts if ev == "display.lease"]
+ assert lease_events and all(holder not in json.dumps(p) for p in lease_events)
+ finally:
+ lease._reset_for_tests()
diff --git a/tests/tui_gateway/test_display_watch.py b/tests/tui_gateway/test_display_watch.py
new file mode 100644
index 0000000000000..b9248724c9d2e
--- /dev/null
+++ b/tests/tui_gateway/test_display_watch.py
@@ -0,0 +1,80 @@
+"""Lease transitions made by ANOTHER process reach `hermes serve` clients as ``display.lease``.
+
+The agent lives in whatever process hosts it (messaging gateway, ``hermes chat``, a cron worker);
+``lease.on_change`` is in-process only, so the serve backend must notice the file move itself.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+
+
+def _other_process(home: Path, stmt: str) -> None:
+ env = {**os.environ, "HERMES_HOME": str(home)}
+ subprocess.run( # noqa: S603
+ [sys.executable, "-c", f"import sys; sys.path.insert(0, {str(REPO)!r}); "
+ f"from tools.bot_desktop import lease; {stmt}"],
+ cwd=str(REPO), env=env, check=True, timeout=60, stdin=subprocess.DEVNULL)
+
+
+def _wait_for(pred, timeout: float = 3.0) -> bool:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if pred():
+ return True
+ time.sleep(0.05)
+ return pred()
+
+
+def _watching(server, home: Path, monkeypatch) -> list:
+ from hermes_constants import hermes_home_key
+ events: list = []
+ monkeypatch.setattr(server, "_broadcast_global_event", lambda ev, payload=None: events.append((ev, payload)))
+ monkeypatch.setattr(server, "_hermes_home", str(home))
+ server._ensure_lease_watcher()
+ assert _wait_for(lambda: hermes_home_key(home) in server._lease_epochs), "watcher never seeded the home"
+ return events
+
+
+def _lease_events(events, **want):
+ return [p for ev, p in events if ev == "display.lease" and all(p["lease"].get(k) == v for k, v in want.items())]
+
+
+def test_handoff_requested_in_another_process_is_broadcast(tmp_path, monkeypatch):
+ import tui_gateway.server as server
+ from hermes_constants import hermes_home_key
+ home = tmp_path / "home"
+ home.mkdir()
+ events = _watching(server, home, monkeypatch)
+
+ _other_process(home, 'lease.request_handoff("probe")')
+
+ assert _wait_for(lambda: _lease_events(events, pending_handoff="probe")), events
+ (payload,) = _lease_events(events, pending_handoff="probe")
+ assert payload["profile_key"] == hermes_home_key(home)
+
+
+def test_release_in_another_process_is_broadcast_and_local_transition_not_duplicated(tmp_path, monkeypatch):
+ import tui_gateway.server as server
+ from tools.bot_desktop import lease
+ home = tmp_path / "home"
+ home.mkdir()
+ events = _watching(server, home, monkeypatch)
+ server._install_lease_listener() # what display.status does: in-process transitions broadcast here
+
+ lease.acquire("viewer-1", profile_key=str(home))
+ assert _wait_for(lambda: _lease_events(events, holder="human"))
+ before = len(events)
+ server._poll_lease_files() # the file moved too, but the watcher saw that epoch locally: no re-broadcast
+ assert len(events) == before
+
+ _other_process(home, 'lease.release("viewer-1")')
+
+ assert _wait_for(lambda: _lease_events(events, holder="agent")), events
+ assert len(_lease_events(events, holder="agent")) == 1
diff --git a/tools/bot_desktop/__init__.py b/tools/bot_desktop/__init__.py
new file mode 100644
index 0000000000000..2ffa400fd64e5
--- /dev/null
+++ b/tools/bot_desktop/__init__.py
@@ -0,0 +1,5 @@
+"""Bot Desktop: per-profile headless Xfce desktop over RFB, viewed from Hermes Desktop.
+
+``runtime`` owns the Xvnc/Xfce process and the published env; ``lease`` owns who may drive the
+screen (agent vs. human); ``rfb_filter`` is the byte-level input gate the WebSocket bridge applies.
+"""
diff --git a/tools/bot_desktop/browser.py b/tools/bot_desktop/browser.py
new file mode 100644
index 0000000000000..08d7489ee3d81
--- /dev/null
+++ b/tools/bot_desktop/browser.py
@@ -0,0 +1,121 @@
+"""The bot's browser on its Bot Desktop: one executable, one persistent user-data-dir per profile.
+
+The agent drives Chromium through agent-browser; a human who takes over clicks the dock's Browser
+icon. Both must be THE SAME browser — same binary, same ``--user-data-dir`` — or the human logs in
+to a jar the bot never sees. Chromium's singleton makes a second launch on the same user-data-dir
+open a window in the running instance, which is exactly the hand-over we want — in ONE direction. When the
+human's dock instance is already up, agent-browser's own launch is forwarded to it and dies without a
+DevTools endpoint, so the dock exposes a debugging port and the agent ATTACHES to it (see
+:func:`running_instance_cdp_port`) instead of launching.
+"""
+
+from __future__ import annotations
+
+import glob
+import os
+import shutil
+import socket
+from pathlib import Path
+from typing import Optional, Tuple
+
+from tools.bot_desktop import runtime
+
+_SYSTEM_BROWSERS = ("google-chrome", "google-chrome-stable", "chromium", "chromium-browser")
+
+
+def profile_dir() -> Path:
+ """User-data-dir the bot's browser uses on this profile's screen (``AGENT_BROWSER_PROFILE`` wins)."""
+ override = os.environ.get("AGENT_BROWSER_PROFILE", "").strip()
+ if override and os.path.isabs(override):
+ return Path(override)
+ return runtime.state_dir() / "browser-profile"
+
+
+def executable() -> Optional[str]:
+ """The Chromium agent-browser launches: an explicit ``AGENT_BROWSER_EXECUTABLE_PATH``, else the newest
+ Playwright Chromium it bundles, else a system Chrome/Chromium. ``None`` when there is none."""
+ explicit = os.environ.get("AGENT_BROWSER_EXECUTABLE_PATH", "").strip()
+ if explicit and os.access(explicit, os.X_OK):
+ return explicit
+ from tools.browser_tool_install import _chromium_search_roots
+ candidates = sorted(
+ (p for root in _chromium_search_roots() for p in glob.glob(os.path.join(root, "chromium-*", "chrome-linux*", "chrome"))),
+ key=os.path.getmtime, reverse=True)
+ for exe in candidates:
+ if os.access(exe, os.X_OK):
+ return exe
+ return next((shutil.which(name) for name in _SYSTEM_BROWSERS if shutil.which(name)), None)
+
+
+def dock_launch() -> Optional[Tuple[str, str]]:
+ """``(executable, user_data_dir)`` for the dock's Browser icon, or ``None`` when no Chromium exists."""
+ exe = executable()
+ return (exe, str(profile_dir())) if exe else None
+
+
+def dock_command(exe: str, user_data_dir: str) -> str:
+ """Shell line the dock's Browser icon runs. ``--remote-debugging-port=0`` makes a human-started
+ instance attachable (Chromium writes the chosen port to ``/DevToolsActivePort``);
+ first-run / default-browser dialogs would sit between the human and the bot's tabs."""
+ return f"{exe} --user-data-dir={user_data_dir} --remote-debugging-port=0 --no-first-run --no-default-browser-check"
+
+
+def running_instance_cdp_port(user_data_dir: str, *, exclude_session: Optional[str] = None) -> Optional[int]:
+ """DevTools port of a Chromium currently running on ``user_data_dir``, or ``None``.
+
+ Both files outlive a crashed or closed Chromium: ``SingletonLock`` is a symlink to ``host-pid`` and
+ ``DevToolsActivePort`` keeps the last port, so the pid must be alive AND the port must accept a
+ connection before it is trusted. An instance agent-browser launched for ``exclude_session`` itself is
+ reported as ``None``: its daemon already owns that browser, and handing it ``--cdp`` would make it
+ close the browser as a config change and then attach to the port that just died with it.
+ """
+ try:
+ with open(os.path.join(user_data_dir, "DevToolsActivePort"), encoding="utf-8") as fh:
+ port_line = fh.readline().strip()
+ target = os.readlink(os.path.join(user_data_dir, "SingletonLock"))
+ except OSError:
+ return None
+ _host, _, pid_text = target.rpartition("-")
+ if not (port_line.isdigit() and pid_text.isdigit()) or not _pid_alive(int(pid_text)):
+ return None
+ if exclude_session and _launched_by_session(int(pid_text)) == exclude_session:
+ return None
+ port = int(port_line)
+ try:
+ with socket.create_connection(("127.0.0.1", port), timeout=0.5):
+ pass
+ except OSError:
+ return None
+ return port
+
+
+def _launched_by_session(chromium_pid: int) -> Optional[str]:
+ """``AGENT_BROWSER_SESSION`` of the agent-browser daemon that spawned ``chromium_pid``, or ``None``
+ for a human-started (dock) instance. Chromium itself gets a scrubbed environment, so the daemon's
+ ``/proc//environ`` is the marker (Linux-only, same user)."""
+ try:
+ with open(f"/proc/{chromium_pid}/status", encoding="utf-8") as fh:
+ ppid = next((int(line.split()[1]) for line in fh if line.startswith("PPid:")), 0)
+ with open(f"/proc/{ppid}/environ", "rb") as fh:
+ raw = fh.read()
+ except (OSError, ValueError):
+ return None
+ for item in raw.split(b"\0"):
+ key, sep, value = item.partition(b"=")
+ if sep and key == b"AGENT_BROWSER_SESSION":
+ return value.decode("utf-8", "replace") or None
+ return None
+
+
+def _pid_alive(pid: int) -> bool:
+ import psutil
+ return psutil.pid_exists(pid)
+
+
+def env_for_agent(env: dict) -> dict:
+ """Pin agent-browser to the screen's browser identity unless the user pinned their own."""
+ env.setdefault("AGENT_BROWSER_PROFILE", str(profile_dir()))
+ exe = executable()
+ if exe:
+ env.setdefault("AGENT_BROWSER_EXECUTABLE_PATH", exe)
+ return env
diff --git a/tools/bot_desktop/install.py b/tools/bot_desktop/install.py
new file mode 100644
index 0000000000000..ee44b7e06b5c9
--- /dev/null
+++ b/tools/bot_desktop/install.py
@@ -0,0 +1,125 @@
+"""Install the Bot Desktop packages on the gateway host from a Desktop client.
+
+The install runs the distro command from ``runtime.install_command()`` (apt/dnf/pacman) as a child
+process on THIS host. Privilege comes from the same masked ``sudo.request`` card the terminal tool
+raises: ``sudo -n true`` is probed first (NOPASSWD / cached timestamp hosts never see a prompt); when
+a password is needed the caller-supplied ``ask_password`` blocks on the card and the value is written
+to sudo's stdin (``-S``) exactly once, never logged, never placed on the command line. Output lines
+stream through ``on_line`` so the pane can show apt's progress; the return value is the exit code.
+
+One install per profile at a time; a second request while one runs is refused.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import os
+import shlex
+import signal
+import subprocess
+import threading
+from typing import Callable, Optional
+
+from hermes_constants import hermes_home_key
+from tools.bot_desktop import runtime
+
+logger = logging.getLogger(__name__)
+
+_install_lock = threading.Lock()
+_running: set[str] = set()
+
+
+class InstallBusy(RuntimeError):
+ pass
+
+
+def assert_not_running() -> None:
+ with _install_lock:
+ if hermes_home_key() in _running:
+ raise InstallBusy("an install is already running for this profile")
+
+
+def claim() -> str:
+ """Atomically take this profile's install slot; raises :class:`InstallBusy` when taken. A caller that
+ claims before handing off to a worker passes ``claimed=True`` to :func:`install_packages`, which then
+ owns releasing it — a check-then-spawn pair (``assert_not_running`` + later claim on the worker) lets
+ two Install clicks both pass the check."""
+ key = hermes_home_key()
+ with _install_lock:
+ if key in _running:
+ raise InstallBusy("an install is already running for this profile")
+ _running.add(key)
+ return key
+
+
+def release(key: str) -> None:
+ with _install_lock:
+ _running.discard(key)
+
+
+def install_packages(*, ask_password: Callable[[], str], on_line: Callable[[str], None],
+ timeout_seconds: float = 900.0, claimed: bool = False) -> int:
+ """Run the package install; returns the process exit code (0 = success, ``-1`` = cancelled).
+ ``claimed=True``: the caller already holds the slot via :func:`claim`; it is released here either way."""
+ key = hermes_home_key() if claimed else None
+ try:
+ if not runtime.is_supported_host():
+ raise RuntimeError("Bot Desktop runs on Linux gateway hosts only")
+ cmd = runtime.install_command()
+ if cmd is None:
+ raise RuntimeError("no supported package manager (apt-get, dnf, pacman) found on this host")
+ if key is None:
+ key = claim()
+ return _run(cmd, ask_password=ask_password, on_line=on_line, timeout_seconds=timeout_seconds)
+ finally:
+ if key is not None:
+ release(key)
+
+
+def _sudo_nopasswd() -> bool:
+ try:
+ return subprocess.run(["sudo", "-n", "true"], capture_output=True, timeout=3,
+ stdin=subprocess.DEVNULL).returncode == 0
+ except Exception:
+ return False
+
+
+def _run(cmd: str, *, ask_password: Callable[[], str], on_line: Callable[[str], None],
+ timeout_seconds: float) -> int:
+ argv = shlex.split(cmd)
+ assert argv[0] == "sudo", cmd
+ stdin_payload: Optional[str] = None
+ if not _sudo_nopasswd():
+ password = ask_password() or ""
+ if not password:
+ on_line("install cancelled: no sudo password provided")
+ return -1
+ # -S: read the password from stdin; -p '': no prompt text mixed into the streamed output.
+ argv = ["sudo", "-S", "-p", "", *argv[1:]]
+ stdin_payload = password + "\n"
+ on_line(f"$ {cmd}")
+ env = {"DEBIAN_FRONTEND": "noninteractive", "LC_ALL": "C.UTF-8"}
+ proc = subprocess.Popen( # windows-footgun: ok — Linux-only (is_supported_host)
+ argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ env={**os.environ, **env}, text=True, encoding="utf-8", errors="replace", start_new_session=True)
+ try:
+ if stdin_payload is not None:
+ proc.stdin.write(stdin_payload) # type: ignore[union-attr]
+ proc.stdin.close() # type: ignore[union-attr]
+ except OSError:
+ pass
+ # The package manager runs in its own session (start_new_session); killing only sudo would leave apt/dnf
+ # running as root with the dpkg lock while the slot is released, so the whole group goes.
+ def _kill_group() -> None:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(proc.pid, signal.SIGKILL) # windows-footgun: ok — Linux-only (is_supported_host)
+
+ timer = threading.Timer(timeout_seconds, _kill_group)
+ timer.start()
+ try:
+ for line in proc.stdout: # type: ignore[union-attr]
+ on_line(line.rstrip("\n"))
+ return proc.wait()
+ finally:
+ timer.cancel()
diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh
new file mode 100755
index 0000000000000..9eb106b1ed710
--- /dev/null
+++ b/tools/bot_desktop/launcher.sh
@@ -0,0 +1,266 @@
+#!/usr/bin/env bash
+# Hermes Bot Desktop — one headless Xfce desktop per Hermes profile, served over RFB.
+#
+# Spawned by tools/bot_desktop/runtime.py with HERMES_BD_* variables set. Runs TigerVNC's Xvnc
+# (X server + RFB server in one process; damage-driven, resizable via SetDesktopSize) listening on a
+# 0600 Unix socket only, then a minimal Xfce started component-wise under a private dbus session.
+#
+# Why not startxfce4 / xfce4-session: xfce4-session expects a logind session scope; outside one it
+# spawns polkit agents that pop empty dialogs and light-locker/xfce4-screensaver lock the desktop for a
+# user who has no password. Starting xfsettingsd -> xfwm4 -> xfdesktop -> xfce4-panel directly, with
+# the screensaver/locker/power-manager autostarts masked, is the shape every headless-VNC recipe
+# converges on (TigerVNC #1096/#581, OpenOnDemand's apptainer desktop, the Arch wiki).
+#
+# Why not the xfce4 metapackage: it drags in the screensaver, power manager and polkit agent this
+# script exists to keep out.
+set -euo pipefail
+
+: "${HERMES_BD_PROFILE:?}" # profile name (display name in the VNC title)
+: "${HERMES_BD_DISPLAY_NUM:?}" # allocated by runtime.py
+: "${HERMES_BD_SOCKET:?}" # RFB unix socket path
+: "${HERMES_BD_XAUTH:?}" # Xauthority path
+: "${HERMES_BD_ENV_FILE:?}" # where to publish DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS
+: "${HERMES_BD_CONFIG_HOME:?}" # per-profile XDG_CONFIG_HOME (xfconf lives here)
+GEOM="${HERMES_BD_GEOMETRY:-1440x900}"
+DEPTH=24
+
+export XDG_CONFIG_HOME="$HERMES_BD_CONFIG_HOME"
+export XDG_CACHE_HOME="${HERMES_BD_CACHE_HOME:-$HERMES_BD_CONFIG_HOME/.cache}"
+export XDG_DATA_HOME="${HERMES_BD_DATA_HOME:-$HERMES_BD_CONFIG_HOME/.local-share}"
+export XDG_SESSION_TYPE=x11 XDG_CURRENT_DESKTOP=XFCE
+export GDK_BACKEND=x11 QT_QPA_PLATFORM=xcb NO_AT_BRIDGE=1 GTK_A11Y=none
+export LANG="${LANG:-C.UTF-8}"
+# Inheriting a login session's bus/session manager yields "Another session manager is already
+# running" / "Unable to contact settings server".
+unset SESSION_MANAGER DBUS_SESSION_BUS_ADDRESS DISPLAY XAUTHORITY WAYLAND_DISPLAY
+
+mkdir -p "$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" "$XDG_CONFIG_HOME/autostart" \
+ "$XDG_CACHE_HOME" "$XDG_DATA_HOME" "$(dirname "$HERMES_BD_SOCKET")"
+
+export DISPLAY=":$HERMES_BD_DISPLAY_NUM"
+export XAUTHORITY="$HERMES_BD_XAUTH"
+
+# Stale lock files from a crashed server block restart; a lock whose pid is alive belongs to a
+# running server (another profile may have taken this number) and is never touched — Xvnc then
+# fails to start on it and runtime.py reports that instead of us disrupting the other desktop.
+rm -f "$HERMES_BD_SOCKET"
+xlock="/tmp/.X${HERMES_BD_DISPLAY_NUM}-lock"
+if [[ -e "$xlock" ]] && ! kill -0 "$(tr -d ' ' < "$xlock" 2>/dev/null)" 2>/dev/null; then
+ rm -f "$xlock" "/tmp/.X11-unix/X${HERMES_BD_DISPLAY_NUM}"
+fi
+: > "$XAUTHORITY"; chmod 600 "$XAUTHORITY"
+xauth -q -f "$XAUTHORITY" add "$DISPLAY" MIT-MAGIC-COOKIE-1 "$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')"
+
+# ---- look: dark theme from whatever the host ships (first match wins), Hermes wallpaper ----
+pick_theme() { local d t; for t in "$@"; do for d in /usr/share/themes "$HOME/.themes"; do [[ -d "$d/$t" ]] && { echo "$t"; return; }; done; done; echo "$1"; }
+pick_icons() { local d t; for t in "$@"; do for d in /usr/share/icons "$HOME/.icons"; do [[ -d "$d/$t" ]] && { echo "$t"; return; }; done; done; echo "$1"; }
+GTK_THEME_NAME=$(pick_theme Adwaita-dark Breeze-Dark Greybird-dark Arc-Dark Adwaita)
+WM_THEME_NAME=$(pick_theme Default-hdpi Default) # xfwm4 window themes ship with xfwm4 itself
+ICON_THEME_NAME=$(pick_icons Papirus-Dark breeze-dark Adwaita hicolor)
+: "${HERMES_BD_WALLPAPER:="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/wallpaper.png"}"
+
+# ---- pre-seed xfconf BEFORE xfconfd starts (it caches; edits after start are overwritten) ----
+X="$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml"
+[[ -e "$X/xfwm4.xml" ]] || cat > "$X/xfwm4.xml" <<'EOF'
+
+
+
+
+
+
+
+
+
+
+EOF
+sed -i "s|HERMES_BD_WM_THEME|$WM_THEME_NAME|" "$X/xfwm4.xml"
+[[ -e "$X/xfce4-screensaver.xml" ]] || cat > "$X/xfce4-screensaver.xml" <<'EOF'
+
+
+
+
+
+EOF
+[[ -e "$X/xsettings.xml" ]] || cat > "$X/xsettings.xml" <<'EOF'
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+EOF
+sed -i "s|HERMES_BD_GTK_THEME|$GTK_THEME_NAME|; s|HERMES_BD_ICON_THEME|$ICON_THEME_NAME|" "$X/xsettings.xml"
+[[ -e "$X/xfce4-desktop.xml" ]] || cat > "$X/xfce4-desktop.xml" <<'EOF'
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+EOF
+sed -i "s|HERMES_BD_WALLPAPER_PLACEHOLDER|$HERMES_BD_WALLPAPER|" "$X/xfce4-desktop.xml"
+# Own panel layout (a layout on disk also suppresses the first-run "Welcome to the panel" dialog):
+# top bar = menu · tasks · tray · clock; bottom dock = only launchers whose program exists on this
+# host, the browser pinned to the one the bot drives so a human lands in the bot's own browser profile.
+if [[ ! -e "$X/xfce4-panel.xml" ]]; then
+ L="$XDG_CONFIG_HOME/xfce4/panel"; mkdir -p "$L"
+ dock_ids=(); n=20
+ add_launcher() { # name icon exec — skipped when the executable is missing
+ local exe; exe=${3%% *}
+ command -v "$exe" >/dev/null 2>&1 || return 0
+ n=$((n+1)); mkdir -p "$L/launcher-$n"
+ printf '[Desktop Entry]\nVersion=1.0\nType=Application\nName=%s\nIcon=%s\nExec=%s\nTerminal=false\nStartupNotify=false\n' \
+ "$1" "$2" "$3" > "$L/launcher-$n/hermes.desktop"
+ dock_ids+=("$n")
+ }
+ add_launcher "Terminal" utilities-terminal "xfce4-terminal"
+ # The bot's browser: runtime.py resolves the executable agent-browser drives plus the profile's
+ # persistent user-data-dir, so a human taking over lands in the bot's own cookie jar.
+ [[ -n "${HERMES_BD_BROWSER_EXEC:-}" ]] && add_launcher "Browser" internet-web-browser "$HERMES_BD_BROWSER_EXEC"
+ add_launcher "Files" system-file-manager "thunar"
+ add_launcher "Text Editor" accessories-text-editor "mousepad"
+ dock_plugins=""; dock_items=""
+ for id in "${dock_ids[@]}"; do
+ dock_plugins+=""
+ dock_items+=""
+ done
+ cat > "$X/xfce4-panel.xml" <
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${dock_plugins}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${dock_items}
+
+
+PANEL
+fi
+# Mask system autostarts that want logind/polkit/keyring/at-spi.
+for a in xfce4-screensaver light-locker xfce4-power-manager xfce-polkit \
+ polkit-gnome-authentication-agent-1 lxpolkit xfce4-notifyd blueman at-spi-dbus-bus \
+ gnome-keyring-pkcs11 gnome-keyring-secrets gnome-keyring-ssh xdg-user-dirs; do
+ [[ -e "$XDG_CONFIG_HOME/autostart/$a.desktop" ]] || \
+ printf '[Desktop Entry]\nType=Application\nName=%s\nHidden=true\n' "$a" > "$XDG_CONFIG_HOME/autostart/$a.desktop"
+done
+# Tests seed the config tree on a fake PATH and stop here (no X server needed).
+[[ -n "${HERMES_BD_SEED_ONLY:-}" ]] && exit 0
+
+# ---- X server + RFB (TigerVNC Xvnc), Unix socket only ----
+# SecurityTypes None is safe ONLY because -rfbport -1 disables TCP and the 0600 socket is reachable
+# only by processes running as this user (the gateway's WebSocket bridge does the real authentication;
+# same-UID processes, the bot's own terminal tool included, are inside that boundary by design).
+# -SendCutText=0: watchers must never receive the holder's clipboard; -AcceptCutText stays on so
+# paste INTO the screen keeps working.
+Xvnc "$DISPLAY" -geometry "$GEOM" -depth "$DEPTH" -dpi 96 \
+ -rfbport -1 -rfbunixpath "$HERMES_BD_SOCKET" -rfbunixmode 0600 \
+ -SecurityTypes None -AlwaysShared -AcceptSetDesktopSize -FrameRate 30 -SendCutText=0 \
+ -desktop "hermes:$HERMES_BD_PROFILE" -auth "$XAUTHORITY" -nolisten tcp \
+ -Log '*:stderr:30' &
+XVNC_PID=$!
+trap 'kill "$XVNC_PID" 2>/dev/null || true' EXIT
+for _ in $(seq 1 100); do
+ xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 && break
+ kill -0 "$XVNC_PID" 2>/dev/null || { echo "Xvnc exited during startup" >&2; exit 1; }
+ sleep 0.1
+done
+xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 || { echo "Xvnc did not become ready" >&2; exit 1; }
+
+setxkbmap -display "$DISPLAY" us 2>/dev/null || true # RFB keysyms + xdotool assume a known layout
+xsetroot -display "$DISPLAY" -solid '#1c1f29' 2>/dev/null || true
+xset -display "$DISPLAY" s off -dpms s noblank 2>/dev/null || true
+
+# ---- private session bus + Xfce components (no xfce4-session) ----
+# dbus-run-session scopes the bus to this subshell: no leaked dbus-daemons on restart. The env file
+# is written from INSIDE the bus so DBUS_SESSION_BUS_ADDRESS is the real one; runtime.py and every
+# cua-driver / browser spawn for this profile source it.
+# Not exec'd: this script stays the supervisor so the EXIT trap above still reaps Xvnc when the Xfce
+# session dies on its own (exec would replace the trap's owner and orphan the X server).
+dbus-run-session -- bash -c '
+ set -e
+ umask 077
+ printf "DISPLAY=%s\nXAUTHORITY=%s\nDBUS_SESSION_BUS_ADDRESS=%s\nXDG_CONFIG_HOME=%s\nXDG_CACHE_HOME=%s\nXDG_DATA_HOME=%s\n" \
+ "$DISPLAY" "$XAUTHORITY" "$DBUS_SESSION_BUS_ADDRESS" "$XDG_CONFIG_HOME" "$XDG_CACHE_HOME" "$XDG_DATA_HOME" \
+ > "$HERMES_BD_ENV_FILE.tmp" && mv -f "$HERMES_BD_ENV_FILE.tmp" "$HERMES_BD_ENV_FILE"
+ xfsettingsd --sm-client-disable --daemon 2>/dev/null || true
+ xfwm4 --compositor=off --sm-client-disable &
+ for _ in $(seq 1 50); do xprop -root _NET_SUPPORTING_WM_CHECK >/dev/null 2>&1 && break; sleep 0.1; done
+ xfdesktop --sm-client-disable --disable-wm-check &
+ exec xfce4-panel --sm-client-disable --disable-wm-check
+' && rc=0 || rc=$?
+kill "$XVNC_PID" 2>/dev/null || true
+exit "$rc"
diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py
new file mode 100644
index 0000000000000..33b12f2009706
--- /dev/null
+++ b/tools/bot_desktop/lease.py
@@ -0,0 +1,243 @@
+"""Who may drive a profile's Bot Desktop screen: the agent (default) or exactly one human viewer.
+
+The lease is the single truth shared by the RFB bridge (drops human input from non-holders), the
+``computer_use`` tool (refuses to act while a human holds control — the person may be typing a
+credential, so even screenshots are refused; fail closed rather than trusting the agent to pause
+itself) and the Desktop UI (Watch / Take over / Hand back).
+
+Authority lives ON DISK, ``/bot-desktop/lease.json`` under an fcntl lock, because the
+processes that must agree do not share memory: ``hermes serve`` (viewer bridge), the messaging
+gateway, a CLI turn and isolated workers all drive the same display. Every read goes to the file;
+the in-process Condition only wakes local waiters early. ``epoch`` increments on every transition so
+an action admitted under one lease can tell that control changed underneath it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Callable, Dict, List, Optional
+
+from hermes_constants import get_hermes_home, hermes_home_key
+
+try:
+ import fcntl
+except ImportError: # Windows/macOS without fcntl: computer_use imports this module on every call, and no
+ fcntl = None # multi-process Bot Desktop exists there, so the cross-process lock degrades to a no-op.
+
+AGENT = "agent"
+HUMAN = "human"
+_POLL_SECONDS = 0.25
+
+
+class HumanHasControl(RuntimeError):
+ """Raised by screen-driving tools while a human holds the lease."""
+
+
+@dataclass
+class Lease:
+ holder: str = AGENT
+ viewer_id: Optional[str] = None
+ since: float = field(default_factory=time.time)
+ reason: str = ""
+ pending_handoff: Optional[str] = None # agent's reason for asking, until the human takes over
+ epoch: int = 0
+
+ def as_dict(self) -> Dict[str, object]:
+ return asdict(self)
+
+
+_lock = threading.Condition()
+_listeners: List[Callable[[str, Lease], None]] = []
+
+
+def _path(profile_key: Optional[str]) -> Path:
+ """``profile_key`` is the HERMES_HOME path of the profile whose lease is meant (the RFB bridge
+ serves several profiles from one process); ``None`` means the current profile."""
+ home = Path(profile_key) if profile_key else get_hermes_home()
+ return home / "bot-desktop" / "lease.json"
+
+
+def _read(path: Path) -> Lease:
+ """No file = fresh profile, agent holds. A file that exists but cannot be parsed is a torn write
+ or tampering: fail CLOSED (human holds) — an unreadable lease must never let the agent act on a
+ screen a human may be using; the next successful write repairs it."""
+ try:
+ raw = path.read_text(encoding="utf-8")
+ except FileNotFoundError:
+ return Lease()
+ except OSError:
+ return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file unreadable")
+ try:
+ data = json.loads(raw)
+ except ValueError:
+ data = None
+ if not isinstance(data, dict) or data.get("holder") not in (AGENT, HUMAN):
+ return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file corrupt")
+ try:
+ return Lease(**{k: v for k, v in data.items() if k in Lease.__dataclass_fields__})
+ except TypeError:
+ return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file corrupt")
+
+
+def _write(path: Path, lease: Lease) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(".json.tmp")
+ tmp.write_text(json.dumps(lease.as_dict()), encoding="utf-8")
+ os.replace(tmp, path)
+
+
+class _locked:
+ """Cross-process critical section over the lease file (fcntl on a sibling lock file)."""
+
+ def __init__(self, path: Path):
+ self._lockfile = path.with_suffix(".lock")
+ self._fh = None
+
+ def __enter__(self):
+ if fcntl is None:
+ return self
+ self._lockfile.parent.mkdir(parents=True, exist_ok=True)
+ self._fh = open(self._lockfile, "a+", encoding="utf-8") # noqa: SIM115 — closed in __exit__
+ fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX)
+ return self
+
+ def __exit__(self, *exc):
+ if self._fh is None:
+ return
+ fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)
+ self._fh.close()
+
+
+def get(profile_key: Optional[str] = None) -> Lease:
+ return _read(_path(profile_key))
+
+
+def on_change(listener: Callable[[str, Lease], None]) -> Callable[[], None]:
+ """Subscribe to lease transitions made IN THIS PROCESS (the gateway broadcasts them to Desktop
+ clients). Transitions made by another process are observed by reading, not by callback."""
+ with _lock:
+ _listeners.append(listener)
+
+ def _off() -> None:
+ with _lock:
+ if listener in _listeners:
+ _listeners.remove(listener)
+ return _off
+
+
+def _notify(key: str, lease: Lease) -> None:
+ for cb in list(_listeners):
+ try:
+ cb(key, lease)
+ except Exception: # a broken subscriber must not wedge the handoff
+ pass
+
+
+def _transition(profile_key: Optional[str], mutate: Callable[[Lease], bool]) -> Lease:
+ key, path = hermes_home_key(profile_key) if profile_key else hermes_home_key(), _path(profile_key)
+ with _locked(path):
+ lease = _read(path)
+ if not mutate(lease):
+ return lease
+ lease.epoch += 1
+ _write(path, lease)
+ with _lock:
+ _lock.notify_all()
+ _notify(key, lease)
+ return lease
+
+
+def acquire(viewer_id: str, *, profile_key: Optional[str] = None, reason: str = "") -> Lease:
+ """Human ``viewer_id`` takes control. Last writer wins: a second viewer evicts the first, and the
+ RFB bridge closes the evicted socket so its UI drops to view-only."""
+ def _m(lease: Lease) -> bool:
+ # The agent's ask ("please log in to X") stays as the takeover reason: the human needs it
+ # on screen WHILE they act, not only before they clicked Take over.
+ lease.holder, lease.viewer_id, lease.since = HUMAN, viewer_id, time.time()
+ lease.reason = reason or lease.pending_handoff or ""
+ lease.pending_handoff = None
+ return True
+ return _transition(profile_key, _m)
+
+
+def release(viewer_id: Optional[str] = None, *, profile_key: Optional[str] = None) -> Lease:
+ """Return control to the agent. With ``viewer_id`` only that holder may release (a stale viewer
+ closing its window must not yank control from the one who took over after it)."""
+ def _m(lease: Lease) -> bool:
+ if viewer_id is not None and lease.holder == HUMAN and lease.viewer_id != viewer_id:
+ return False
+ lease.holder, lease.viewer_id, lease.since, lease.reason = AGENT, None, time.time(), ""
+ lease.pending_handoff = None # "hand back" answers an open request even if nobody formally took over
+ return True
+ return _transition(profile_key, _m)
+
+
+def request_handoff(reason: str, *, profile_key: Optional[str] = None) -> Lease:
+ """Agent asks a human to take over (login, 2FA, CAPTCHA, payment). Recorded so the UI can show
+ why and the bridge can page the user; control itself still flips only on ``acquire``."""
+ def _m(lease: Lease) -> bool:
+ lease.pending_handoff = reason
+ return True
+ return _transition(profile_key, _m)
+
+
+def wait_for_release(*, timeout: float, profile_key: Optional[str] = None) -> bool:
+ """Block until the agent holds the lease (and no handoff is pending) or ``timeout`` elapses.
+ True when control is back with the agent. Polls the file so a release made by another process
+ is seen; the local Condition just shortens the wait for same-process transitions."""
+ return _wait_until(lambda lease: lease.holder == AGENT and lease.pending_handoff is None, timeout, profile_key)
+
+
+def wait_for_takeover_or_release(*, timeout: float, profile_key: Optional[str] = None) -> bool:
+ """False when, after ``timeout``, the agent still holds with a handoff pending: nobody answered."""
+ return _wait_until(lambda lease: lease.holder == HUMAN or lease.pending_handoff is None, timeout, profile_key)
+
+
+def _wait_until(done: Callable[[Lease], bool], timeout: float, profile_key: Optional[str]) -> bool:
+ path = _path(profile_key)
+ deadline = time.monotonic() + timeout
+ while True:
+ if done(_read(path)):
+ return True
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return False
+ with _lock:
+ _lock.wait(min(remaining, _POLL_SECONDS))
+
+
+def human_holds(profile_key: Optional[str] = None) -> bool:
+ return get(profile_key).holder == HUMAN
+
+
+def viewer_may_send_input(viewer_id: str, *, profile_key: Optional[str] = None) -> bool:
+ lease = get(profile_key)
+ return lease.holder == HUMAN and lease.viewer_id == viewer_id
+
+
+def assert_agent_may_act(profile_key: Optional[str] = None) -> Lease:
+ """The lease as of now, or ``HumanHasControl``. Callers keep the returned ``epoch`` and compare it
+ with ``get().epoch`` after an admitted action: a change means a human took over mid-flight."""
+ lease = get(profile_key)
+ if lease.holder == HUMAN:
+ raise HumanHasControl(
+ "A human has taken over this desktop (they may be entering a credential). Screen actions and "
+ "captures are refused until they hand control back; call computer_use action='wait_for_human' "
+ "to block until then.")
+ return lease
+
+
+def _reset_for_tests() -> None:
+ with _lock:
+ _listeners.clear()
+ p = get_hermes_home() / "bot-desktop" / "lease.json"
+ for f in (p, p.with_suffix(".lock"), p.with_suffix(".json.tmp")):
+ try:
+ f.unlink()
+ except OSError:
+ pass
diff --git a/tools/bot_desktop/rfb_filter.py b/tools/bot_desktop/rfb_filter.py
new file mode 100644
index 0000000000000..b4d4a63e2c426
--- /dev/null
+++ b/tools/bot_desktop/rfb_filter.py
@@ -0,0 +1,100 @@
+"""Byte-level RFB client→server gate for the Bot Desktop WebSocket bridge.
+
+noVNC's ``viewOnly`` is a UI hint; anyone holding the socket could still inject input. The bridge
+parses the client stream and forwards only non-input messages from viewers that do not hold the
+lease. RFB messages do not align with WebSocket frames, so this is a stateful stream parser fed
+arbitrary chunks (RFC 6143 §7.5 layouts; TigerVNC's EnableContinuousUpdates 150 and Fence 248 pass
+through untouched since they carry no input; QEMU Extended KeyEvent 255 is keyboard input — noVNC
+switches to it as soon as Xvnc advertises the pseudo-encoding — so it is gated like KeyEvent).
+
+Xvnc runs ``-SecurityTypes None``, so the handshake is fixed-size: 12-byte version, 1-byte security
+choice, then ``ClientInit`` (1 byte). ``ServerInit`` is server→client and never crosses this filter.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+_INPUT_TYPES = {4, 5, 6, 255} # KeyEvent, PointerEvent, ClientCutText, QEMU Extended KeyEvent
+
+# Fixed-length client messages: type -> total length including the type byte.
+_FIXED = {
+ 0: 20, # SetPixelFormat
+ 3: 10, # FramebufferUpdateRequest
+ 4: 8, # KeyEvent
+ 5: 6, # PointerEvent
+ 150: 10, # EnableContinuousUpdates
+ 255: 12, # QEMU client message; sub-type 0 = Extended KeyEvent (the only one noVNC sends)
+}
+_SET_ENCODINGS = 2
+_CLIENT_CUT_TEXT = 6
+_FENCE = 248
+
+# TigerVNC's default MaxCutText. The length is client-declared (int32); without a cap a watcher
+# with a ticket but no lease could make the bridge buffer ~2 GiB waiting for a payload.
+_MAX_CUT_TEXT = 256 * 1024
+
+
+class RfbClientFilter:
+ """Feed client bytes with :meth:`feed`; get back the bytes allowed to reach Xvnc.
+
+ ``allow_input`` is consulted per message so a lease flip mid-stream applies to the very next
+ key or pointer event.
+ """
+
+ def __init__(self, allow_input: Callable[[], bool]) -> None:
+ self._allow_input = allow_input
+ self._buf = bytearray()
+ self._handshake_left = 12 + 1 + 1 # version + security type + ClientInit(shared flag)
+
+ def feed(self, chunk: bytes) -> bytes:
+ self._buf += chunk
+ out = bytearray()
+ if self._handshake_left:
+ take = min(self._handshake_left, len(self._buf))
+ if take:
+ head = bytes(self._buf[:take])
+ # ClientInit shared-flag: force shared so a human viewer never disconnects the agent's
+ # watcher or another observer (Xvnc also runs -AlwaysShared; belt and braces at zero cost).
+ if self._handshake_left - take == 0 and take >= 1:
+ head = head[:-1] + b"\x01"
+ out += head
+ del self._buf[:take]
+ self._handshake_left -= take
+ if self._handshake_left:
+ return bytes(out)
+ while self._buf:
+ length = self._message_length()
+ if length is None or len(self._buf) < length:
+ break
+ msg = bytes(self._buf[:length])
+ del self._buf[:length]
+ if msg[0] in _INPUT_TYPES and not self._allow_input():
+ continue
+ out += msg
+ return bytes(out)
+
+ def _message_length(self) -> int | None:
+ t = self._buf[0]
+ if t in _FIXED:
+ return _FIXED[t]
+ if t == _SET_ENCODINGS:
+ if len(self._buf) < 4:
+ return None
+ n = int.from_bytes(self._buf[2:4], "big")
+ return 4 + 4 * n
+ if t == _CLIENT_CUT_TEXT:
+ if len(self._buf) < 8:
+ return None
+ n = int.from_bytes(self._buf[4:8], "big", signed=True)
+ # Extended clipboard (RFB 3.8 + TigerVNC): negative length, |n| bytes follow.
+ if abs(n) > _MAX_CUT_TEXT:
+ raise ValueError("clipboard message too large")
+ return 8 + abs(n)
+ if t == _FENCE:
+ if len(self._buf) < 9:
+ return None
+ return 9 + self._buf[8]
+ # Unknown client message: we cannot frame it, and forwarding blind would let an input message
+ # hide behind it. Drop the rest of the stream; the viewer reconnects.
+ raise ValueError(f"unknown RFB client message type {t}")
diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py
new file mode 100644
index 0000000000000..a9ce1923f0993
--- /dev/null
+++ b/tools/bot_desktop/runtime.py
@@ -0,0 +1,385 @@
+"""Bot Desktop runtime: one headless Xfce desktop per Hermes profile, served over RFB on a private
+Unix socket, viewed and driven from Hermes Desktop.
+
+Layout under ``/bot-desktop/``: ``display`` (allocated X display number), ``rfb.sock``
+(Xvnc RFB Unix socket, 0600), ``Xauthority``, ``env`` (DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS
+published by the launcher once Xfce's bus exists), ``launcher.pid``, ``launcher.log``, ``xdg/``
+(per-profile XDG_CONFIG_HOME so two profiles never share xfconf). Everything is profile-scoped via
+``get_hermes_home()`` so N profiles in one gateway get N desktops: one screen per bot on the shared
+machine.
+
+The launcher is ``launcher.sh`` next to this module; :func:`desktop_env` is what cua-driver and headed
+Chromium spawns merge in so the agent acts on this profile's screen and nowhere else.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import os
+import shutil
+import signal
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional
+
+from hermes_constants import get_hermes_home
+
+logger = logging.getLogger(__name__)
+
+_LAUNCHER = Path(__file__).with_name("launcher.sh")
+
+# Display numbers below 10 collide with real seats and default Xvfb recipes (:99 is popular too); scan a
+# private band and record the choice so restarts reuse it.
+_DISPLAY_MIN, _DISPLAY_MAX = 20, 89
+
+# Binaries the launcher execs; the package hint is per distro family.
+REQUIRED_BINARIES = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session",
+ "xauth", "xdpyinfo", "setxkbmap", "xprop")
+
+# Which package in each distro list ships each required binary. Fedora retired the xorg-x11-utils /
+# xorg-x11-server-utils umbrellas (per-binary packages since F35) and dnf5 refuses the whole transaction on
+# one unknown name, so every binary must map to a package that still resolves; the test suite checks that
+# each mapped package is in PACKAGES for its manager.
+BINARY_PACKAGES = {
+ "apt": {"Xvnc": "tigervnc-standalone-server", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel",
+ "xfdesktop": "xfdesktop4", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11",
+ "xauth": "xauth", "xdpyinfo": "x11-utils", "setxkbmap": "x11-xkb-utils", "xprop": "x11-utils"},
+ "dnf": {"Xvnc": "tigervnc-server-minimal", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel",
+ "xfdesktop": "xfdesktop", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11",
+ "xauth": "xorg-x11-xauth", "xdpyinfo": "xdpyinfo", "setxkbmap": "setxkbmap", "xprop": "xprop"},
+ "pacman": {"Xvnc": "tigervnc", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", "xfdesktop": "xfdesktop",
+ "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus",
+ "xauth": "xorg-xauth", "xdpyinfo": "xorg-xdpyinfo", "setxkbmap": "xorg-setxkbmap", "xprop": "xorg-xprop"},
+}
+
+PACKAGES = {
+ "apt": ["tigervnc-standalone-server", "xfce4-panel", "xfwm4", "xfdesktop4", "xfce4-settings",
+ "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "x11-xkb-utils", "xauth",
+ "fonts-dejavu-core"],
+ "dnf": ["tigervnc-server-minimal", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings",
+ "xfce4-terminal", "dbus-x11", "xsetroot", "xset", "xdpyinfo", "xprop", "xorg-x11-xauth", "setxkbmap",
+ "dejavu-sans-fonts"],
+ "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", "dbus",
+ "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xprop", "xorg-xauth", "xorg-setxkbmap",
+ "ttf-dejavu"],
+}
+
+
+def state_dir() -> Path:
+ return get_hermes_home() / "bot-desktop"
+
+
+def is_supported_host() -> bool:
+ return sys.platform.startswith("linux")
+
+
+def missing_binaries() -> list[str]:
+ return [b for b in REQUIRED_BINARIES if shutil.which(b) is None]
+
+
+def package_manager() -> Optional[str]:
+ for pm in ("apt-get", "dnf", "pacman"):
+ if shutil.which(pm):
+ return "apt" if pm == "apt-get" else pm
+ return None
+
+
+def install_command() -> Optional[str]:
+ pm = package_manager()
+ if pm is None:
+ return None
+ pkgs = " ".join(PACKAGES[pm])
+ return {
+ "apt": f"sudo apt-get install -y --no-install-recommends {pkgs}",
+ "dnf": f"sudo dnf install -y {pkgs}",
+ "pacman": f"sudo pacman -S --needed --noconfirm {pkgs}",
+ }[pm]
+
+
+@dataclass
+class DesktopStatus:
+ profile: str
+ supported: bool
+ installed: bool
+ missing: list[str]
+ running: bool
+ pid: Optional[int]
+ display: Optional[str]
+ socket: Optional[str]
+ geometry: str
+ install_command: Optional[str]
+
+ def as_dict(self) -> Dict[str, object]:
+ return dict(self.__dict__)
+
+
+def _read(path: Path) -> Optional[str]:
+ try:
+ return path.read_text(encoding="utf-8").strip() or None
+ except OSError:
+ return None
+
+
+def _pid_alive(pid: int) -> bool:
+ import psutil
+ return psutil.pid_exists(pid)
+
+
+def _create_time(pid: int) -> Optional[float]:
+ import psutil
+ try:
+ return psutil.Process(pid).create_time()
+ except (psutil.Error, OverflowError, ValueError):
+ return None
+
+
+def _launcher_pid() -> Optional[int]:
+ """The live launcher's pid, or None. ``launcher.pid`` holds ``" "``: a recycled pid
+ with a different start time is somebody else's process and must never be reported as ours nor
+ killed by :func:`stop`. The pre-identity single-number format is treated as not running."""
+ raw = _read(state_dir() / "launcher.pid")
+ pid_s, _, born_s = (raw or "").partition(" ")
+ if not pid_s.isdigit() or not born_s:
+ return None
+ try:
+ pid, born = int(pid_s), float(born_s)
+ except ValueError:
+ return None
+ actual = _create_time(pid)
+ return pid if actual is not None and abs(actual - born) < 0.01 else None
+
+
+_X_LOCK_DIR = Path("/tmp") # where X servers write .X-lock (tests point it at a scratch dir)
+
+
+def _display_in_use(num: int) -> bool:
+ """A live X server owns ``:num``: its lock file names a running pid. A lock left by a crashed
+ server (dead pid) does not count, so the number can be reclaimed."""
+ lock = _X_LOCK_DIR / f".X{num}-lock"
+ try:
+ pid = int(lock.read_text(encoding="utf-8").strip())
+ except (OSError, ValueError):
+ return False
+ return _pid_alive(pid)
+
+
+_ALLOC_LOCK = Path("/tmp/.hermes-bot-desktop-alloc.lock") # host-wide: profiles allocate from one band
+
+
+@contextlib.contextmanager
+def _flocked(path: Path):
+ import fcntl # windows-footgun: ok — Linux-only runtime (is_supported_host gates start)
+ with open(path, "a+", encoding="utf-8") as fh: # windows-footgun: ok — Linux-only runtime
+ fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
+ try:
+ yield fh
+ finally:
+ fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
+
+
+def _pick_display() -> int:
+ """Caller holds ``_ALLOC_LOCK``. The recorded number is only reused when no OTHER server holds it now:
+ after profile A stops, B may have taken A's number, and A's launcher must never unlink B's socket."""
+ recorded = _read(state_dir() / "display")
+ if recorded and recorded.isdigit() and not _display_in_use(int(recorded)):
+ return int(recorded)
+ for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1):
+ if not _display_in_use(num):
+ return num
+ raise RuntimeError("no free X display number in the Bot Desktop band")
+
+
+def _allocate_display() -> int:
+ with _flocked(_ALLOC_LOCK):
+ return _pick_display()
+
+
+def desktop_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+ """``base_env`` (default ``os.environ``) with this profile's DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS
+ merged in when its desktop is running. Unchanged otherwise, so hosts with a real seat keep it.
+ Pure: never starts anything (it is called from env builders, status probes and tests)."""
+ env = dict(os.environ if base_env is None else base_env)
+ published = published_env()
+ if published:
+ env.update(published)
+ env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends
+ from tools.bot_desktop.browser import env_for_agent
+ env_for_agent(env) # same binary + user-data-dir as the dock's Browser icon
+ return env
+
+
+def ensure_started_for_tool() -> None:
+ """Tool-boundary hook (``computer_use`` dispatch): with ``bot_desktop.auto_start`` (opt-in, default off) a Linux
+ host that has NO display and the packages installed gets its screen started on first use, so a headless
+ gateway works the first time instead of answering "no DISPLAY is set". Failure is not an error here;
+ the tool's own "no display" diagnosis is the right message then."""
+ if published_env() or not _should_auto_start(os.environ):
+ return
+ try:
+ start()
+ except Exception as exc:
+ logger.info("Bot Desktop auto-start skipped: %s", exc)
+
+
+def _should_auto_start(env: Dict[str, str]) -> bool:
+ if not is_supported_host() or env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"):
+ return False
+ if missing_binaries():
+ return False
+ from hermes_cli.config import load_config_readonly
+ cfg = load_config_readonly().get("bot_desktop") or {}
+ return bool(cfg.get("auto_start", False))
+
+
+def published_env() -> Dict[str, str]:
+ """Variables the launcher wrote once Xfce's private bus existed; empty when the desktop is down."""
+ if _launcher_pid() is None:
+ return {}
+ raw = _read(state_dir() / "env")
+ if not raw:
+ return {}
+ out: Dict[str, str] = {}
+ for line in raw.splitlines():
+ key, sep, value = line.partition("=")
+ if sep:
+ out[key.strip()] = value.strip()
+ return out
+
+
+def rfb_socket_path() -> Optional[Path]:
+ sock = state_dir() / "rfb.sock"
+ return sock if _launcher_pid() is not None and sock.exists() else None
+
+
+def geometry() -> str:
+ from hermes_cli.config import load_config_readonly
+ cfg = load_config_readonly().get("bot_desktop") or {}
+ return str(cfg.get("geometry") or "1440x900")
+
+
+def status(profile: Optional[str] = None) -> DesktopStatus:
+ missing: list[str] = missing_binaries() if is_supported_host() else list(REQUIRED_BINARIES)
+ pid = _launcher_pid()
+ env = published_env()
+ return DesktopStatus(
+ profile=profile or _profile_name(),
+ supported=is_supported_host(),
+ installed=not missing,
+ missing=missing,
+ running=pid is not None and bool(env.get("DISPLAY")),
+ pid=pid,
+ display=env.get("DISPLAY"),
+ socket=str(rfb_socket_path()) if rfb_socket_path() else None,
+ geometry=geometry(),
+ install_command=install_command() if missing else None,
+ )
+
+
+def _profile_name() -> str:
+ try:
+ from hermes_cli.profiles import get_active_profile_name
+ return get_active_profile_name() or "default"
+ except Exception:
+ return "default"
+
+
+def start(*, wait_seconds: float = 15.0) -> DesktopStatus:
+ """Start this profile's desktop (idempotent). Blocks until the launcher publishes its env file or
+ ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker.
+
+ Two locks, both held from the running-check to the launcher's publish: the per-profile ``start.lock``
+ so two start() calls for one profile spawn one launcher (the loser sees it running), and the host-wide
+ display-allocation lock so a second profile cannot pick the same number before this Xvnc has written
+ ``/tmp/.X-lock`` (it would then fail and its launcher's stale-lock cleanup could remove our socket)."""
+ if not is_supported_host():
+ raise RuntimeError("Bot Desktop runs on Linux gateway hosts only")
+ missing = missing_binaries()
+ if missing:
+ hint = install_command() or "install TigerVNC (Xvnc) and the Xfce core components"
+ raise RuntimeError(f"Bot Desktop needs {', '.join(missing)} on the gateway host. Install: {hint}")
+ sd = state_dir()
+ sd.mkdir(parents=True, exist_ok=True)
+ os.chmod(sd, 0o700)
+ with _flocked(sd / "start.lock"):
+ if _launcher_pid() is not None and published_env().get("DISPLAY"):
+ return status()
+ with _flocked(_ALLOC_LOCK):
+ return _spawn_and_wait(sd, _pick_display(), wait_seconds)
+
+
+def _spawn_and_wait(sd: Path, num: int, wait_seconds: float) -> DesktopStatus:
+ (sd / "display").write_text(str(num), encoding="utf-8")
+ env_file = sd / "env"
+ env_file.unlink(missing_ok=True)
+
+ child_env = {k: v for k, v in os.environ.items() if k not in {
+ "DISPLAY", "XAUTHORITY", "WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER"}}
+ child_env.update({
+ "HERMES_BD_PROFILE": _profile_name(),
+ "HERMES_BD_DISPLAY_NUM": str(num),
+ "HERMES_BD_SOCKET": str(sd / "rfb.sock"),
+ "HERMES_BD_XAUTH": str(sd / "Xauthority"),
+ "HERMES_BD_ENV_FILE": str(env_file),
+ "HERMES_BD_CONFIG_HOME": str(sd / "xdg"),
+ "HERMES_BD_GEOMETRY": geometry(),
+ })
+ from tools.bot_desktop.browser import dock_command, dock_launch
+ if (browser := dock_launch()) is not None:
+ child_env["HERMES_BD_BROWSER_EXEC"] = dock_command(*browser)
+ # Truncated per start: the log is a diagnostic for THIS launch, and nothing rotates it otherwise.
+ log = open(sd / "launcher.log", "wb") # noqa: SIM115 — handed to the child, closed by it
+ proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host)
+ ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
+ start_new_session=True, close_fds=True)
+ log.close()
+ born = _create_time(proc.pid)
+ (sd / "launcher.pid").write_text(f"{proc.pid} {born if born is not None else 0}", encoding="utf-8")
+
+ deadline = time.monotonic() + wait_seconds
+ while time.monotonic() < deadline:
+ if proc.poll() is not None:
+ tail = (sd / "launcher.log").read_bytes()[-2000:].decode("utf-8", "replace")
+ raise RuntimeError(f"Bot Desktop launcher exited with {proc.returncode}:\n{tail}")
+ if env_file.exists() and (sd / "rfb.sock").exists():
+ logger.info("Bot Desktop for profile %s up on :%s", _profile_name(), num)
+ return status()
+ time.sleep(0.1)
+ raise RuntimeError(f"Bot Desktop did not publish its display within {wait_seconds:.0f}s (see {sd / 'launcher.log'})")
+
+
+def stop() -> bool:
+ """Stop this profile's desktop; True when a running launcher was signalled."""
+ if not is_supported_host():
+ return False
+ sd = state_dir()
+ sd.mkdir(parents=True, exist_ok=True)
+ with _flocked(sd / "start.lock"):
+ return _stop_locked(sd)
+
+
+def _stop_locked(sd: Path) -> bool:
+ pid = _launcher_pid()
+ if pid is None:
+ (sd / "env").unlink(missing_ok=True)
+ return False
+ # The launcher runs in its own session; killing the group takes Xvnc, dbus and Xfce with it.
+ try:
+ os.killpg(pid, signal.SIGTERM) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start)
+ except ProcessLookupError:
+ pass
+ for _ in range(50):
+ if not _pid_alive(pid):
+ break
+ time.sleep(0.1)
+ else:
+ try:
+ os.killpg(pid, signal.SIGKILL) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start)
+ except ProcessLookupError:
+ pass
+ (sd / "launcher.pid").unlink(missing_ok=True)
+ (sd / "env").unlink(missing_ok=True)
+ return True
diff --git a/tools/bot_desktop/thumbnail.py b/tools/bot_desktop/thumbnail.py
new file mode 100644
index 0000000000000..eeb61b57892b0
--- /dev/null
+++ b/tools/bot_desktop/thumbnail.py
@@ -0,0 +1,46 @@
+"""Thumbnail of a bot's screen: one JPEG grab of the profile's Xvnc display.
+
+Feeds the Screen hero in Hermes Desktop (the big preview at the top of a bot's pane). Read-only:
+it never touches the lease, so a human in control is not disturbed and the bot is not blocked.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import os
+import threading
+from typing import Optional
+
+from tools.bot_desktop import runtime
+
+THUMB_MAX = (960, 600)
+_grab_lock = threading.Lock()
+
+
+def thumbnail_data_url(max_size: tuple[int, int] = THUMB_MAX, quality: int = 72) -> Optional[str]:
+ """``data:image/jpeg;base64,...`` of the running screen, or ``None`` when no screen is up."""
+ env = runtime.published_env()
+ display = env.get("DISPLAY")
+ if not display or runtime._launcher_pid() is None:
+ return None
+ from PIL import ImageGrab # Pillow is a hard dependency; import lazily to keep status calls cheap
+
+ # Xlib reads XAUTHORITY from the process env; the launcher publishes a per-profile cookie file.
+ # The swap is process-wide, so two profiles grabbed on worker threads at once serialise here or
+ # one would grab with the other's cookie and restore the wrong value.
+ with _grab_lock:
+ previous = os.environ.get("XAUTHORITY")
+ if env.get("XAUTHORITY"):
+ os.environ["XAUTHORITY"] = env["XAUTHORITY"]
+ try:
+ image = ImageGrab.grab(xdisplay=display)
+ finally:
+ if previous is None:
+ os.environ.pop("XAUTHORITY", None)
+ else:
+ os.environ["XAUTHORITY"] = previous
+ image.thumbnail(max_size)
+ buf = io.BytesIO()
+ image.convert("RGB").save(buf, "JPEG", quality=quality, optimize=True)
+ return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode("ascii")
diff --git a/tools/bot_desktop/wallpaper.png b/tools/bot_desktop/wallpaper.png
new file mode 100644
index 0000000000000..3d933f827d282
Binary files /dev/null and b/tools/bot_desktop/wallpaper.png differ
diff --git a/tools/browser_tool.py b/tools/browser_tool.py
index 104bc7694df44..99af9a0a07089 100644
--- a/tools/browser_tool.py
+++ b/tools/browser_tool.py
@@ -42,7 +42,9 @@ def _build_browser_env() -> dict:
env = hermes_subprocess_env(inherit_credentials=False)
env.update({k: os.environ[k] for k in _BROWSER_PASSTHROUGH_KEYS if k in os.environ})
- return env
+ # Headed Chromium opens on this profile's Bot Desktop when one is running (human can take it over).
+ from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env
+ return _bot_desktop_env(env)
try:
@@ -817,7 +819,9 @@ def _json_with_fallback(response: Dict[str, Any], result: Dict[str, Any]) -> str
def _failed_response(result: Dict[str, Any], default_error: str) -> str:
- return _json_with_fallback(_err(result.get("error", default_error)), result)
+ # ``code`` = machine-readable refusal (human_has_control), same shape as computer_use's.
+ extra = {"code": result["code"]} if result.get("code") else {}
+ return _json_with_fallback(_err(result.get("error", default_error), **extra), result)
def _tool_response(result: Dict[str, Any], ok: Dict[str, Any], default_error: str) -> str:
diff --git a/tools/browser_tool_real_profile.py b/tools/browser_tool_real_profile.py
index 103bc7a64cc7f..dfbe6f0e4a025 100644
--- a/tools/browser_tool_real_profile.py
+++ b/tools/browser_tool_real_profile.py
@@ -163,12 +163,13 @@ def _launch_real_profile_chrome(real_binary: str, copy_dir: str) -> Tuple[Option
except OSError:
pass
chrome_argv = [real_binary, f"--user-data-dir={copy_dir}", *_REAL_PROFILE_CHROME_FLAGS]
- _has_display = bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
+ browser_env = _bt._build_browser_env() # carries the Bot Desktop DISPLAY when one is running
+ _has_display = bool(browser_env.get("DISPLAY") or browser_env.get("WAYLAND_DISPLAY"))
if not (_cloud._is_headed_mode() and (_has_display or not sys.platform.startswith("linux"))):
chrome_argv.append("--headless=new")
try:
chrome_proc = subprocess.Popen(chrome_argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
- stdin=subprocess.DEVNULL, start_new_session=True, env=_bt._build_browser_env())
+ stdin=subprocess.DEVNULL, start_new_session=True, env=browser_env)
except (subprocess.SubprocessError, OSError) as e:
return None, f"{_RP}the launch failed: {e}"
_bt._real_profile_chrome_procs.append(chrome_proc)
diff --git a/tools/browser_tool_session.py b/tools/browser_tool_session.py
index 4b9b353b3e034..e762d191075ef 100644
--- a/tools/browser_tool_session.py
+++ b/tools/browser_tool_session.py
@@ -574,6 +574,47 @@ def _run_browser_command(
except Exception as e:
_bt.logger.warning("Failed to create browser session for task=%s: %s", task_id, e)
return {"success": False, "error": f"Failed to create browser session: {str(e)}"}
+ # The bot's LOCAL browser lives on its Bot Desktop screen, in the same profile a human who took
+ # over is typing into. While the human holds the lease every action AND read against it is
+ # refused (the page may show their credential); the fence brackets the whole run so a takeover
+ # mid-command also voids the result. Cloud / user-supplied CDP sessions are a different browser.
+ if _shares_bot_desktop_browser(session_info):
+ from tools.bot_desktop import lease as _bd_lease
+ try:
+ admitted = _bd_lease.assert_agent_may_act()
+ except _bd_lease.HumanHasControl as e:
+ return {"success": False, "error": str(e), "code": "human_has_control"}
+ result = _run_browser_command_unfenced(task_id, command, args, timeout, _engine_override, browser_cmd, session_info)
+ if _bd_lease.get().epoch != admitted.epoch:
+ return {"success": False, "code": "human_has_control",
+ "error": "A human took over the bot's screen while this browser command ran; its result was "
+ "discarded. Call computer_use action='wait_for_human' to block until they hand back."}
+ return result
+ return _run_browser_command_unfenced(task_id, command, args, timeout, _engine_override, browser_cmd, session_info)
+
+
+def _shares_bot_desktop_browser(session_info: Dict[str, Any]) -> bool:
+ """Decided by provenance, not transport: every LOCAL session (plain ``--session``, real-profile CDP
+ attach, Lightpanda) is a browser Hermes launched with this profile's Bot Desktop DISPLAY, so it is the
+ screen a human who took over is typing into. Cloud / user-supplied CDP sessions are another browser.
+ A human lease with the screen already gone (dead Xvnc) still fences — computer_use does the same."""
+ if not (session_info.get("features") or {}).get("local"):
+ return False
+ from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime
+ return bool(_bd_runtime.published_env().get("DISPLAY")) or _bd_lease.human_holds()
+
+
+def _bot_desktop_attach_port(session_info: Dict[str, Any]) -> Optional[int]:
+ """DevTools port of a human-started Chromium on the Bot Desktop's shared profile, else ``None``."""
+ if not _shares_bot_desktop_browser(session_info):
+ return None
+ from tools.bot_desktop import browser as _bd_browser
+ return _bd_browser.running_instance_cdp_port(str(_bd_browser.profile_dir()),
+ exclude_session=session_info["session_name"])
+
+
+def _run_browser_command_unfenced(task_id: str, command: str, args: List[str], timeout: int,
+ _engine_override: Optional[str], browser_cmd, session_info: Dict[str, Any]) -> Dict[str, Any]:
# Cleanup stops the supervisor before closing the backend; keep it stopped.
if command != "close" and session_info.get("cdp_url"):
_cdp._ensure_cdp_supervisor(task_id)
@@ -587,6 +628,12 @@ def _run_browser_command(
backend_args = ["--cdp", session_info["cdp_url"]]
else:
backend_args = ["--session", session_info["session_name"]]
+ if (bd_port := _bot_desktop_attach_port(session_info)) is not None:
+ # A Chromium already runs on the Bot Desktop's shared profile (the human clicked the dock's
+ # Browser first): a launch would be forwarded into it by Chromium's singleton and die without
+ # a DevTools endpoint, so the session's daemon attaches to the port it advertises instead.
+ # Same daemon (keyed by --session) either way, so snapshot refs stay valid across commands.
+ backend_args += ["--cdp", str(bd_port)]
if _cloud._is_headed_mode():
backend_args.append("--headed")
if engine != "auto" and not _bt._is_camofox_mode():
diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py
index 9be4b19cb77ea..d6075d2e7aab1 100644
--- a/tools/computer_use/cua_backend.py
+++ b/tools/computer_use/cua_backend.py
@@ -100,12 +100,28 @@ def _computer_use_max_image_dimension() -> Optional[int]:
dim = 1456
return dim if dim > 0 else None
+def desktop_identity(env: Optional[Dict[str, str]] = None) -> str:
+ """The screen a backend spawned from ``env`` acts on: its DISPLAY (``''`` when none). Recorded next to the
+ cached backend so a Bot Desktop that starts (or restarts on another number) AFTER the backend was cached is
+ noticed — the cached cua-driver still points at the old seat or at no display at all."""
+ return str((cua_driver_child_env(env) if env is None else env).get("DISPLAY") or "")
+
+
+def backend_display_stale(recorded: str, current: str) -> bool:
+ """True when a cached backend's recorded display identity no longer matches the one a fresh spawn would get."""
+ return (recorded or "") != (current or "")
+
+
def cua_driver_child_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]:
"""Env for spawning cua-driver: ``base_env`` (default ``os.environ``) plus ``CUA_DRIVER_RS_TELEMETRY_ENABLED=0``
unless the user opted in, plus the native-Wayland bridge (``computer_use.native_wayland`` config opt-in, only when
the child has a Wayland display). Used by every spawn site (MCP, status, doctor, install) so CLI and gateway
runtimes share one policy."""
env = dict(os.environ if base_env is None else base_env)
+ # A running Bot Desktop for this profile owns the agent's screen: DISPLAY/XAUTHORITY/DBUS point there so
+ # cua-driver never acts on a seat the human is sitting at (#90374 class) and headless hosts get a display.
+ from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env
+ env = _bot_desktop_env(env)
if _cua_telemetry_disabled():
env[_CUA_TELEMETRY_ENV_VAR] = "0"
if sys.platform == "linux" and env.get("WAYLAND_DISPLAY") and bool(_computer_use_cfg().get("native_wayland", False)):
diff --git a/tools/computer_use/handoff.py b/tools/computer_use/handoff.py
new file mode 100644
index 0000000000000..bdd38acbb1ad0
--- /dev/null
+++ b/tools/computer_use/handoff.py
@@ -0,0 +1,50 @@
+"""Human handoff actions for ``computer_use`` on a Bot Desktop: ``request_handoff`` asks the person to
+take over the screen (login, 2FA, CAPTCHA, payment) and ``wait_for_human`` blocks until control is
+back. Both are answered without touching cua-driver, so a human typing a credential is never
+captured.
+
+The Desktop pane shows the request (the gateway broadcasts the lease change). Reaching the person
+anywhere else is the model's job: it relays the ask in its reply, which is what lands in the chat
+surface the user is actually on. This module only records intent on the lease and waits.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict
+
+from tools.bot_desktop import lease as _lease
+
+HANDOFF_ACTIONS = frozenset({"request_handoff", "wait_for_human"})
+_DEFAULT_WAIT_SECONDS = 600.0
+_MAX_WAIT_SECONDS = 1800.0
+_DEFAULT_GRACE_SECONDS = 60.0
+
+
+def handle_handoff(action: str, args: Dict[str, Any]) -> str:
+ if action == "request_handoff":
+ reason = str(args.get("reason") or "The agent needs you to complete a step on its screen.").strip()
+ _lease.request_handoff(reason)
+ return json.dumps({
+ "ok": True, "action": action, "state": _lease.get().as_dict(),
+ "next": "Hermes Desktop now shows 'Bot needs you' on this bot's Screen. Tell the user in your "
+ "reply what to do and that they can take over from Bots > Screen, then call "
+ "computer_use action='wait_for_human' to block until they hand control back and "
+ "re-capture before continuing — the screen state is whatever they left."})
+ timeout = min(_MAX_WAIT_SECONDS, max(1.0, float(args.get("seconds") or _DEFAULT_WAIT_SECONDS)))
+ grace = min(timeout, max(0.0, float(args.get("grace") or _DEFAULT_GRACE_SECONDS)))
+ # Nobody has taken over yet: give them `grace` to click Take over, then return so the model can chase
+ # the user in chat instead of blocking the whole timeout on a request nobody saw. Once a human holds
+ # the screen, wait the full timeout for the hand-back.
+ if not _lease.wait_for_takeover_or_release(timeout=grace):
+ state = _lease.get().as_dict()
+ return json.dumps({"ok": False, "action": action, "code": "no_takeover", "state": state,
+ "error": f"Nobody took over within {grace:.0f}s. Ask the user in chat to open Bots > Screen and "
+ "click Take over, then call wait_for_human again."})
+ released = _lease.wait_for_release(timeout=timeout)
+ state = _lease.get().as_dict()
+ if released:
+ return json.dumps({"ok": True, "action": action, "state": state,
+ "next": "Control is back with you. Take a fresh capture; do not assume prior state."})
+ return json.dumps({"ok": False, "action": action, "code": "still_waiting", "state": state,
+ "error": f"No hand-back within {timeout:.0f}s. Call wait_for_human again or ask the user in chat."})
diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py
index a38943c142bba..880d9bdc58273 100644
--- a/tools/computer_use/schema.py
+++ b/tools/computer_use/schema.py
@@ -32,12 +32,16 @@
"list_apps",
"list_windows",
"focus_app",
+ "request_handoff",
+ "wait_for_human",
],
"description": (
"Which action to perform. `capture` is free (no side effects). All other actions "
"require approval unless auto-approved. Use `set_value` for select/popup elements and "
"sliders — it selects the matching option directly without opening the native menu (no "
- "focus steal)."
+ "focus steal). When a login, 2FA, CAPTCHA or payment step needs the human, call "
+ "`request_handoff` (with `reason`) so they can take over this screen from the Hermes "
+ "Desktop app, then `wait_for_human`; while they hold control every other action is refused."
),
},
"mode": {
@@ -141,13 +145,15 @@
),
},
"text": {"type": "string", "description": "Text to type (respects the current layout)."},
+ "reason": {"type": "string", "description": "request_handoff: one sentence telling the human what to do on the screen (e.g. 'Sign in to LinkedIn and complete 2FA')."},
"keys": {
"type": "string",
"description": (
"Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine."
),
},
- "seconds": {"type": "number", "description": "Seconds to wait. Max 30."},
+ "seconds": {"type": "number", "description": "wait: seconds to pause (max 30). wait_for_human: how long to block for the hand-back (default 600, max 1800)."},
+ "grace": {"type": "number", "description": "wait_for_human: seconds to wait for someone to take over before returning no_takeover (default 60); once a human holds control the full `seconds` applies."},
"raise_window": {
"type": "boolean",
"description": (
diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py
index afb000913f173..32f892db38c16 100644
--- a/tools/computer_use/tool.py
+++ b/tools/computer_use/tool.py
@@ -79,6 +79,7 @@ def _input_target_mismatch(backend, requested_app: str) -> Optional[str]:
_backends: Dict[str, ComputerUseBackend] = {}
_backend_call_locks: Dict[str, threading.RLock] = {}
_backend_permission_modes: Dict[str, str] = {}
+_backend_displays: Dict[str, str] = {} # DISPLAY the cached backend was spawned against (Bot Desktop rebind)
# (home key, provider, model) → bool. The decision reads the active profile's config (auxiliary.vision
# override, declared supports_vision), so a multiplexed process must not serve profile A's verdict to B.
_AUX_VISION_ROUTE_CACHE: Dict[Tuple[str, str, str], bool] = {}
@@ -133,7 +134,9 @@ def _install_backend(sid: str, backend: ComputerUseBackend, permission_mode: str
"""Record a backend in the session caches (the empty session also mirrors it onto the ``_backend`` hook).
Caller holds ``_backend_lock``."""
global _backend
+ from tools.computer_use.cua_backend import desktop_identity
_backends[sid], _backend_permission_modes[sid] = backend, permission_mode
+ _backend_displays[sid] = desktop_identity()
_backend_call_locks[sid] = threading.RLock()
_backend = backend if sid == "" else _backend
return backend
@@ -142,7 +145,7 @@ def _detach_locked(sid: str) -> Tuple[Optional[ComputerUseBackend], Optional[thr
"""Remove one session's cache entries, plus the ``_backend`` injection hook when it aliases the empty session
(older callers/tests may populate only the hook). Caller holds ``_backend_lock``."""
global _backend
- _backend_permission_modes.pop(sid, None)
+ _backend_permission_modes.pop(sid, None), _backend_displays.pop(sid, None)
backend, call_lock = _backends.pop(sid, None), _backend_call_locks.pop(sid, None)
if sid == "":
backend = _backend if backend is None else backend
@@ -170,9 +173,12 @@ def _get_backend(session_id: str = "") -> ComputerUseBackend:
backend = _new_backend(permission_mode)
backend.start() # under the cache lock: one backend per session; a concurrent toggle releases it
return _install_backend(sid, backend, permission_mode)
- if _backend_permission_modes.get(sid, "standard") == permission_mode:
+ from tools.computer_use.cua_backend import backend_display_stale, desktop_identity
+ if (_backend_permission_modes.get(sid, "standard") == permission_mode
+ and not backend_display_stale(_backend_displays.get(sid, ""), desktop_identity())):
return cached
- # Cua's mode is immutable after daemon startup: a /yolo toggle replaces only this session's backend.
+ # Cua's mode and DISPLAY are fixed at daemon startup: a /yolo toggle, or a Bot Desktop that started
+ # (or moved) after this backend was cached, replaces only this session's backend.
_, stale_lock = _detach_locked(sid) # stopped outside the cache lock; the loop re-reads the mode first
_stop_backend(cached, stale_lock, lambda e: None)
@@ -207,7 +213,7 @@ def _shutdown_backend_atexit() -> None:
if _backend is not None:
unique.setdefault(id(_backend), (_backend, _backend_call_locks.get("")))
_backend = None
- _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear()
+ _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear(), _backend_displays.clear()
with _approval_lock:
_session_auto_approve.clear(), _always_allow.clear(), _escalation_warned.clear()
for backend, call_lock in unique.values():
@@ -248,6 +254,19 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any:
if not action:
return json.dumps({"error": "missing `action`"})
session_id = str(kwargs.get("session_id") or "") # approval-state / daemon-mode isolation key
+ from tools.computer_use.handoff import HANDOFF_ACTIONS, handle_handoff
+ if action in HANDOFF_ACTIONS:
+ return handle_handoff(action, args)
+ # Bot Desktop lease: while a human drives the screen every action, capture included, is refused.
+ from tools.bot_desktop import lease as _bd_lease
+ from tools.bot_desktop.runtime import ensure_started_for_tool as _bd_ensure_started
+ def _refused(e: Exception) -> str:
+ return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)})
+ try:
+ admitted = _bd_lease.assert_agent_may_act()
+ except _bd_lease.HumanHasControl as e:
+ return _refused(e)
+ _bd_ensure_started() # headless gateway: bring the profile's screen up before the backend probes DISPLAY
if (err := _reject_unsafe(action, args)) is not None:
return err
scopes = ([action] if action in _ACTIONS and _ACTIONS[action].destructive else []) + (
@@ -265,7 +284,27 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any:
with _backend_lock:
call_lock = _backend_call_locks.setdefault(session_id, threading.RLock())
with call_lock:
- return _dispatch(backend, action, args)
+ # Re-check under the dispatch lock: approval, backend start-up and lock waits above can take
+ # seconds, and a human may have taken over meanwhile. A result produced after such a flip is
+ # discarded too — it may picture what they typed.
+ try:
+ _bd_lease.assert_agent_may_act()
+ except _bd_lease.HumanHasControl as e:
+ return _refused(e)
+
+ def _fence() -> None:
+ # Any lease transition since admission voids the frame — including a full take-over /
+ # hand-back cycle that already finished: it still belongs to the human's turn. Capture
+ # paths call this BEFORE the frame is persisted, spilled or sent to auxiliary vision.
+ if _bd_lease.get().epoch != admitted.epoch:
+ raise _bd_lease.HumanHasControl(
+ "A human took over this desktop while the action ran; its result was discarded. "
+ "Re-capture (or call computer_use action='wait_for_human' if they still hold control).")
+ result = _dispatch(backend, action, args, fence=_fence)
+ _fence()
+ return result
+ except _bd_lease.HumanHasControl as e:
+ return _refused(e)
except Exception as e:
logger.exception("computer_use %s failed", action)
return json.dumps({"error": f"{action} failed: {e}"})
@@ -334,12 +373,13 @@ def _do_scroll(backend, action, args, **delivery):
return backend.scroll(direction=args.get("direction", "down"), amount=int(args.get("amount", 3)),
element=args.get("element"), **_scroll_xy(args), modifiers=args.get("modifiers"), **delivery)
-def _do_capture(backend, action, args, **_):
+def _do_capture(backend, action, args, fence=lambda: None, **_):
if (mode := str(args.get("mode", "som"))) not in {"som", "vision", "ax"}:
return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"})
# pid/window_id forwarded only when given so older backends keep their defaults.
- return _capture_response(backend.capture(mode=mode, app=args.get("app"),
- **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None}))
+ cap = backend.capture(mode=mode, app=args.get("app"), **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None})
+ fence()
+ return _capture_response(cap)
def _do_listing(backend, action, args, key, **_):
return json.dumps({key: (items := getattr(backend, action)()), "count": len(items)})
@@ -389,7 +429,9 @@ def _summarize_click(action: str, args: Dict[str, Any], fg: str) -> str:
"input_text": "type", "screenshot": "capture", "get_window_state": "capture", "left_click": "click", "mouse_click": "click",
}
-def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> Any:
+def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any], fence: Callable[[], None] = lambda: None) -> Any:
+ """``fence`` raises when the screen lease moved since admission; capture paths call it as soon as the
+ frame is in hand, before anything derived from it leaves the process."""
spec = _ACTIONS.get(action)
if spec is None:
return json.dumps({"error": f"unknown action {action!r}" + (f" — did you mean {hint!r}? See the action enum in the tool schema."
@@ -402,10 +444,11 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) ->
f"{action} would go to the current target {mismatch!r}, not {requested_app.strip()!r} "
"— input actions always hit the sticky target from the last capture/focus_app. "
f"Call capture(app={requested_app.strip()!r}) or focus_app first, then retry.")})
- # delivery_mode / bring_to_front thread through every input action (background → foreground ladder).
- res = spec.handler(backend, action, args, delivery_mode=args.get("delivery_mode"),
- bring_to_front=bool(args.get("bring_to_front")))
- return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after")))
+ # delivery_mode / bring_to_front thread through every input action (background → foreground ladder);
+ # read-only actions get the lease fence instead (delivery kwargs would leak into backend input calls).
+ res = spec.handler(backend, action, args, **(dict(delivery_mode=args.get("delivery_mode"), bring_to_front=bool(args.get("bring_to_front")))
+ if spec.input else dict(fence=fence)))
+ return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after")), fence)
# ── Response shaping ────────────────────────────────────────────────────────
def _classify_action_result(res: ActionResult) -> Dict[str, Any]:
@@ -583,7 +626,8 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
"elements_file — read_file/search_files it, or pass app= to narrow scope)")
return _text_capture_payload(v, "\n".join(lines), extra)
-def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool) -> Any:
+def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool,
+ fence: Callable[[], None] = lambda: None) -> Any:
# No follow-up capture after a failed action: a normal-looking screenshot would suggest success.
if not do_capture or not res.ok:
return _text_response(res)
@@ -596,6 +640,7 @@ def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_cap
except Exception as e:
logger.warning("follow-up capture failed: %s", e)
return _text_response(res)
+ fence()
resp, payload = _capture_response(cap), _action_payload(res)
if isinstance(resp, dict) and resp.get("_multimodal"):
# Keep the evidence/verdict contract visible alongside the image — it governs whether input may repeat.
diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py
new file mode 100644
index 0000000000000..660a8a685ff1f
--- /dev/null
+++ b/tui_gateway/methods_display.py
@@ -0,0 +1,221 @@
+"""Bot Desktop JSON-RPC handlers: the Desktop app's door to a profile's headless screen.
+
+``display.status`` reports runtime + lease; ``display.start`` / ``display.stop`` manage the Xvnc/Xfce
+process; ``display.observe`` mints a single-use ticket the renderer redeems on ``/api/display/ws``
+(``hermes_cli.web_routers.display``) to stream raw RFB; ``display.lease.acquire`` / ``release`` are
+Take over / Hand back. ``display.install`` runs the distro package install on the gateway host: sudo
+privilege is asked for through the masked ``display.install.sudo.request`` card (same ``_block`` bridge as
+the terminal tool's sudo prompt), stdout streams as ``display.install.log`` and the run ends with
+``display.install.done`` carrying a fresh status snapshot. Every handler is profile-scoped so a multiplexed gateway answers for the bot
+the pane is looking at. Lease transitions fan out as the global ``display.lease`` event so every
+connected client repaints (badge on the bot row, red border on the viewer, agent handoff prompt).
+
+Bodies are rebound onto server.py's globals (method_ctx.bind_module) and reference them bare.
+"""
+
+import logging
+import threading
+import weakref
+
+from .method_ctx import HandlerRegistry, bind_module
+
+logger = logging.getLogger(__name__)
+_registry = HandlerRegistry()
+method = _registry.method
+_profile_scoped = _registry.profile_scoped
+
+_DISPLAY_ERR = 5300
+_lease_listener_installed = threading.Event()
+
+
+def _lease_view(lease) -> dict:
+ """The lease as clients may see it: the holder's viewer id is a capability (whoever presents it
+ co-drives or releases the lease), so it is replaced by a short hash the holder can match against
+ its own id to know it is the one in control."""
+ import hashlib
+ d = lease.as_dict()
+ vid = d.pop("viewer_id")
+ d["viewer_id"] = None
+ d["viewer_hash"] = hashlib.sha256(vid.encode()).hexdigest()[:12] if vid else None
+ return d
+
+
+def _display_snapshot() -> dict:
+ from hermes_constants import hermes_home_key
+ from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime
+ st = _bd_runtime.status()
+ return {**st.as_dict(), "lease": _lease_view(_bd_lease.get()), "profile_key": hermes_home_key()}
+
+
+def _install_lease_listener() -> None:
+ """Once per process: broadcast every lease change to all connected clients."""
+ if _lease_listener_installed.is_set():
+ return
+ from tools.bot_desktop import lease as _bd_lease
+
+ def _on_change(profile_key: str, lease) -> None:
+ _broadcast_global_event("display.lease", {"profile_key": profile_key, "lease": _lease_view(lease)})
+ _bd_lease.on_change(_on_change)
+ _lease_listener_installed.set() # only once the subscription exists, or a failed import would silence every client
+
+
+@method("display.status")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ _install_lease_listener()
+ try:
+ return _ok(rid, _display_snapshot())
+ except Exception as e:
+ return _err(rid, _DISPLAY_ERR, str(e))
+
+
+@method("display.thumbnail")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ """One JPEG grab of the bot's screen (``data_url``: null while stopped). Read-only: no lease change.
+ Suppressed while a human holds the lease — the frame may show what they are typing."""
+ try:
+ from tools.bot_desktop import lease as _bd_lease
+ if _bd_lease.human_holds():
+ return _ok(rid, {"data_url": None, "suppressed": "human_has_control"})
+ from tools.bot_desktop.thumbnail import thumbnail_data_url
+ return _ok(rid, {"data_url": thumbnail_data_url()})
+ except Exception as e:
+ return _err(rid, _DISPLAY_ERR, str(e))
+
+
+@method("display.start")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ _install_lease_listener()
+ from tools.bot_desktop import runtime as _bd_runtime
+ try:
+ _bd_runtime.start()
+ return _ok(rid, _display_snapshot())
+ except Exception as e:
+ return _err(rid, _DISPLAY_ERR, str(e))
+
+
+@method("display.stop")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime
+ try:
+ _bd_lease.release()
+ stopped = _bd_runtime.stop()
+ return _ok(rid, {**_display_snapshot(), "stopped": stopped})
+ except Exception as e:
+ return _err(rid, _DISPLAY_ERR, str(e))
+
+
+# viewer ids minted per connection (keyed by the transport that asked), so a reconnecting pane can
+# keep its identity — and its lease — while nobody can claim an id minted for another connection.
+_minted_viewer_ids: "weakref.WeakKeyDictionary[object, set[str]]" = weakref.WeakKeyDictionary()
+
+
+def _mint_viewer_id(requested: str) -> str:
+ """Server-minted viewer identity. ``requested`` is honoured only when THIS connection minted it
+ earlier; anything else (including a holder id read off display.status) gets a fresh id."""
+ import secrets
+ try:
+ mine = _minted_viewer_ids.setdefault(current_transport(), set())
+ except TypeError: # stdio / slotted transports cannot be weakly referenced: always mint
+ mine = set()
+ if requested in mine:
+ return requested
+ viewer_id = secrets.token_urlsafe(16)
+ mine.add(viewer_id)
+ return viewer_id
+
+
+@method("display.observe")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ """Mint a single-use, 30 s ticket for ``/api/display/ws``. The ticket carries the profile home so
+ the bridge dials THIS profile's RFB socket, and a server-minted viewer id (returned to the caller,
+ who passes it to ``display.lease.acquire`` / ``release``) so the lease can name the holder."""
+ from hermes_constants import get_hermes_home
+ from hermes_cli.dashboard_auth.ws_tickets import mint_ticket
+ from tools.bot_desktop import runtime as _bd_runtime
+ try:
+ if _bd_runtime.rfb_socket_path() is None:
+ return _err(rid, _DISPLAY_ERR, "this profile's Bot Desktop is not running; call display.start first")
+ viewer_id = _mint_viewer_id(str(params.get("viewer_id") or "").strip())
+ ticket = mint_ticket(user_id=f"display:{viewer_id}", provider="bot-desktop",
+ extra={"hermes_home": str(get_hermes_home()), "viewer_id": viewer_id})
+ return _ok(rid, {"ticket": ticket, "path": "/api/display/ws", "viewer_id": viewer_id,
+ **_display_snapshot()})
+ except Exception as e:
+ return _err(rid, _DISPLAY_ERR, str(e))
+
+
+@method("display.install")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ """Start the package install in the background; the renderer follows ``display.install.log`` /
+ ``display.install.done``. Refused while one is already running for this profile."""
+ from hermes_constants import hermes_home_key
+ from tools.bot_desktop import install as _bd_install, runtime as _bd_runtime
+ if not _bd_runtime.is_supported_host():
+ return _err(rid, _DISPLAY_ERR, "Bot Desktop runs on Linux gateway hosts only")
+ if _bd_runtime.install_command() is None:
+ return _err(rid, _DISPLAY_ERR, "no supported package manager (apt-get, dnf, pacman) on this host")
+ profile_key = hermes_home_key()
+ sid = str(params.get("session_id") or "")
+
+ def _ask_password() -> str:
+ return _block("display.install.sudo.request", sid, {"profile_key": profile_key}, timeout=300)
+
+ def _line(text: str) -> None:
+ _broadcast_global_event("display.install.log", {"profile_key": profile_key, "line": text})
+
+ # The worker thread inherits NO context: the caller's transport (so the sudo card reaches the
+ # CLIENT that clicked Install) and the profile scope `_profile_scoped` installed (so status, lock
+ # and events all speak for the requested profile) are carried across with copy_context().
+ import contextvars
+ ctx = contextvars.copy_context()
+
+ def _run() -> None:
+ try:
+ code = _bd_install.install_packages(ask_password=_ask_password, on_line=_line, claimed=True)
+ except Exception as e:
+ _line(f"install failed: {e}")
+ code = 1
+ _broadcast_global_event("display.install.done", {"profile_key": profile_key, "code": code,
+ "status": _display_snapshot()})
+
+ try:
+ _bd_install.claim() # atomic: two fast clicks cannot both start a package manager
+ except _bd_install.InstallBusy as e:
+ return _err(rid, _DISPLAY_ERR, str(e))
+ threading.Thread(target=ctx.run, args=(_run,), name=f"bot-desktop-install:{profile_key}", daemon=True).start()
+ return _ok(rid, {"started": True, "command": _bd_runtime.install_command(), "profile_key": profile_key})
+
+
+@method("display.lease.acquire")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ from tools.bot_desktop import lease as _bd_lease
+ viewer_id = str(params.get("viewer_id") or "").strip()
+ if not viewer_id:
+ return _err(rid, _DISPLAY_ERR, "viewer_id required")
+ lease = _bd_lease.acquire(viewer_id, reason=str(params.get("reason") or ""))
+ return _ok(rid, {"lease": _lease_view(lease)})
+
+
+@method("display.lease.release")
+@_profile_scoped
+def _(rid, params: dict) -> dict:
+ from tools.bot_desktop import lease as _bd_lease
+ viewer_id = str(params.get("viewer_id") or "").strip() or None
+ # lease.release(None) skips the holder check; a client that lost its viewer id must not be able to
+ # yank control from whoever holds it unless it says so explicitly (force).
+ if viewer_id is None and not params.get("force") and _bd_lease.human_holds():
+ return _err(rid, _DISPLAY_ERR, "viewer_id required to release another viewer's lease (or pass force: true)",
+ data={"code": "viewer_mismatch"})
+ lease = _bd_lease.release(viewer_id)
+ return _ok(rid, {"lease": _lease_view(lease)})
+
+
+def register(server) -> None:
+ bind_module(globals(), server, skip=("_",))
diff --git a/tui_gateway/methods_display_watch.py b/tui_gateway/methods_display_watch.py
new file mode 100644
index 0000000000000..3050d08b7678e
--- /dev/null
+++ b/tui_gateway/methods_display_watch.py
@@ -0,0 +1,87 @@
+"""Cross-process lease watcher: ``display.lease`` for transitions made OUTSIDE ``hermes serve``.
+
+The lease (``tools.bot_desktop.lease``) lives on disk and is changed by whatever process hosts the
+agent — the messaging gateway's ``request_handoff``, a ``hermes chat`` takeover, a cron worker's
+release. ``lease.on_change`` only fires in the writing process, so ``methods_display``'s in-process
+listener never sees those; the Desktop's "Bot needs you" badge and hero tone stayed stale until the
+pane was reopened. One daemon thread stats every served home's ``bot-desktop/lease.json`` (launch
+home + ``_served_profile_homes``) every 0.5s and broadcasts the SAME ``display.lease`` payload when
+the epoch moves. Bodies are rebound onto server.py's globals (method_ctx.bind_module).
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from pathlib import Path
+
+from .method_ctx import bind_module
+
+_LEASE_POLL_S = 0.5
+_lease_watcher_started = threading.Event()
+# profile key → last epoch broadcast or seen (in-process transitions record theirs too, so a change
+# made by THIS process is not re-broadcast when its file write is noticed a tick later).
+_lease_epochs: dict[str, int] = {}
+_lease_mtimes: dict[str, int | None] = {}
+
+
+def _lease_event_payload(profile_key: str, lease) -> dict:
+ # Same shape and the same redaction (viewer_hash, never the raw id) as the in-process broadcast.
+ from tui_gateway.methods_display import _lease_view
+ return {"profile_key": profile_key, "lease": _lease_view(lease)}
+
+
+def _watched_lease_homes() -> list[Path]:
+ return [Path(_hermes_home), *_served_profile_homes]
+
+
+def _poll_lease_files() -> None:
+ """One pass: read a home's lease only when its file mtime moved; broadcast when the epoch did."""
+ from hermes_constants import hermes_home_key
+ from tools.bot_desktop import lease as _bd_lease
+ for home in _watched_lease_homes():
+ key = hermes_home_key(home)
+ try:
+ mtime = (home / "bot-desktop" / "lease.json").stat().st_mtime_ns
+ except OSError:
+ mtime = None
+ if mtime == _lease_mtimes.get(key, 0):
+ continue
+ _lease_mtimes[key] = mtime
+ lease = _bd_lease.get(str(home))
+ if key not in _lease_epochs: # first sighting seeds silently: display.status carries it
+ _lease_epochs[key] = lease.epoch
+ continue
+ if lease.epoch == _lease_epochs[key]:
+ continue
+ _lease_epochs[key] = lease.epoch
+ _broadcast_global_event("display.lease", _lease_event_payload(key, lease))
+
+
+def _ensure_lease_watcher() -> None:
+ """Once per process, from the first display.* call: start the lease-file poll thread and mark
+ in-process transitions as seen so they broadcast exactly once (via the in-process listener)."""
+ if _lease_watcher_started.is_set():
+ return
+ _lease_watcher_started.set()
+ from tools.bot_desktop import lease as _bd_lease
+
+ def _seen_locally(profile_key: str, lease) -> None:
+ # methods_display's listener broadcasts in-process transitions; only then is the file
+ # move ours to skip. Before display.status installed it, the poll below carries them.
+ if _lease_listener_installed.is_set():
+ _lease_epochs[profile_key] = lease.epoch
+ _bd_lease.on_change(_seen_locally)
+
+ def _loop() -> None:
+ while True:
+ try:
+ _poll_lease_files()
+ except Exception: # noqa: BLE001 - a torn read must not kill the watcher
+ logger.debug("lease watcher poll failed", exc_info=True)
+ time.sleep(_LEASE_POLL_S)
+ threading.Thread(target=_loop, name="hermes-lease-watcher", daemon=True).start()
+
+
+def register(server) -> None:
+ bind_module(globals(), server, skip=("_",))
diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py
index bdb0a40b929f0..8824dc3b23c6a 100644
--- a/tui_gateway/methods_prompt.py
+++ b/tui_gateway/methods_prompt.py
@@ -1113,7 +1113,7 @@ def _(rid, params: dict) -> dict:
_LATE_RESPOND_KEYS = {
"terminal.read.respond": "text", "preview.read.respond": "text", "preview.act.respond": "text",
"window.read.respond": "text", "tour.respond": "text", "mcp.setup.respond": "result",
- "sudo.respond": "password", "secret.respond": "value", "vault.unlock.respond": "password",
+ "sudo.respond": "password", "display.install.sudo.respond": "password", "secret.respond": "value", "vault.unlock.respond": "password",
"vault.save_login.respond": "login", "vault.code.respond": "code"}
for _name, _key in _LATE_RESPOND_KEYS.items():
method(_name)(lambda rid, params, _k=_key: _respond(rid, params, _k, allow_expired=True))
diff --git a/tui_gateway/server.py b/tui_gateway/server.py
index cbfe41572dbc9..b5101afc0ba2e 100644
--- a/tui_gateway/server.py
+++ b/tui_gateway/server.py
@@ -1252,7 +1252,7 @@ def _enable_gateway_prompts() -> None:
# Blocking bridges whose `*.respond` tolerates a late reply (allow_expired=True): on timeout the tool
# returns empty, but a slow renderer could still answer and hit a raw 4009 — `.expire` tears the card down.
_EXPIRING_REQUESTS = frozenset({
- "secret.request", "sudo.request", "vault.unlock.request", "vault.save_login.request", "vault.code.request", "clarify.request",
+ "secret.request", "sudo.request", "display.install.sudo.request", "vault.unlock.request", "vault.save_login.request", "vault.code.request", "clarify.request",
"terminal.read.request",
"preview.read.request", "preview.act.request", "window.read.request", "mcp.setup.request",
"tour.request",
@@ -3230,7 +3230,8 @@ def _resolve_name(name: str) -> str:
methods_projects as _methods_projects, methods_session_foreign as _methods_session_foreign,
methods_session_control as _methods_session_control, methods_subagents as _methods_subagents,
methods_vault as _methods_vault, methods_free_tier as _methods_free_tier,
- methods_connectors as _methods_connectors)
+ methods_connectors as _methods_connectors, methods_display as _methods_display,
+ methods_display_watch as _methods_display_watch)
for _m in (
_session_transports, _session_reaper, _session_lifecycle, _session_workdir, _compute_host_bridge, _model_switch,
@@ -3240,6 +3241,7 @@ def _resolve_name(name: str) -> str:
_methods_browser_control, _methods_session, _methods_prompt, _methods_config,
_methods_config_set, _methods_complete, _methods_tools, _methods_profiles, _methods_images,
_methods_bot_relay, _prompt_turn, _billing_view, _methods_projects, _methods_session_foreign,
- _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors):
+ _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors,
+ _methods_display, _methods_display_watch):
_m.register(sys.modules[__name__])
del _m
diff --git a/tui_gateway/ws.py b/tui_gateway/ws.py
index 480ed36636940..448224fab1a86 100644
--- a/tui_gateway/ws.py
+++ b/tui_gateway/ws.py
@@ -304,6 +304,7 @@ def _error(code: int, message: str, req_id: Any) -> dict:
# Live-apply skins Hermes activates mid-conversation, and track this peer for session-less
# global broadcasts write_json can't route.
server._ensure_skin_watcher()
+ server._ensure_lease_watcher() # cross-process lease moves → display.lease
server.register_live_transport(transport)
# Cross-backend liveness: a heartbeat row lets the startup orphan sweep tell "live but idle
# backend" from "truly orphaned". Idempotent and once-per-process, like the orphan sweep (the
diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md
new file mode 100644
index 0000000000000..f201fbf775ee1
--- /dev/null
+++ b/website/docs/user-guide/features/bot-screen.md
@@ -0,0 +1,168 @@
+---
+title: Bot Screen
+sidebar_position: 17
+---
+
+# Bot Screen
+
+On a headless Linux gateway host (a server, a cloud VM, Hermes Cloud) each bot
+gets its **own desktop**: an Xfce screen the bot's `computer_use` and headed
+browser act on, streamed live into Hermes Desktop. Watch what the bot does,
+**take over** when it hits a login, 2FA prompt, CAPTCHA or payment step, then
+**hand control back** and let it continue with the session you just signed in
+to. The bot keeps working after you close the app or turn off your laptop; the
+screen lives on the gateway host, not on your machine.
+
+Every Hermes profile ("bot") has its own screen, its own browser profile and
+its own cookies. Screens are work surfaces, not security boundaries: the bots
+share the host's user account, files and network (the same model as other
+hosted-agent products).
+
+**Threat model.** The screen's RFB socket, the X display, the browser profile
+and the control-lease file all belong to the gateway's OS user. Any process
+running as that user — another bot on the same host, and the bot's own
+`terminal` tool included — can reach them directly, bypassing the pane and the
+lease. The lease is a tool-level fence on `computer_use` and the browser tools,
+not an OS one. Running each bot as its own OS user is out of scope; if that
+isolation matters to you, put the bots on separate hosts.
+
+## Requirements
+
+- The gateway host runs Linux. macOS and Windows hosts already have a real
+ display; the pane is not offered there.
+- TigerVNC's `Xvnc` and the Xfce core components are installed on the host.
+ Nothing installs them silently: `hermes update` and fresh installs leave every
+ machine as it is. When they are missing the Screen pane in Hermes Desktop shows
+ **Install on host** — one click runs the package manager on the gateway host
+ (it asks for that host's sudo password in a masked card; the password goes to
+ that host only and is never stored) and streams the log. From a shell,
+ `hermes computer-use screen status` prints the exact line and
+ `hermes computer-use screen install` runs it:
+
+ | Distro | Packages |
+ |---|---|
+ | Debian / Ubuntu | `tigervnc-standalone-server xfce4-panel xfwm4 xfdesktop4 xfce4-settings xfce4-terminal dbus-x11 x11-xserver-utils x11-utils xauth fonts-dejavu-core` |
+ | Fedora | `tigervnc-server-minimal xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal dbus-x11 xsetroot xset xdpyinfo xprop xorg-x11-xauth setxkbmap dejavu-sans-fonts` |
+ | Arch | `tigervnc xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal xorg-xsetroot xorg-xset xorg-xdpyinfo xorg-xprop xorg-xauth xorg-setxkbmap ttf-dejavu` |
+
+ Deliberately **not** the `xfce4` metapackage: it pulls in the screensaver,
+ power manager and polkit agent that lock or prompt a headless desktop.
+- [Computer Use](./computer-use.md) enabled for the bot (cua-driver installed).
+
+## Using it
+
+Every bot's computer is one click away in three places of Hermes Desktop:
+
+- **Bots → a bot → Scheduled Jobs**: the bot's screen is the hero at the very
+ top of the pane, above the title and the routines: a live preview of the
+ desktop (refreshed every few seconds while the pane is visible) with who holds
+ control; click the picture to expand into live access. While the screen is off
+ or not installed the same box says so and offers Start / Install.
+- **Bots → right-click a bot → Open Screen**.
+- **Sessions sidebar**, grouped by gateway / profile: the same **Screen** box
+ sits under each profile's header, so a profile's machine is reachable from
+ its conversations too.
+
+1. Open the Screen with any of the entries above.
+ The first time, click **Start screen**. Set `bot_desktop.auto_start: true` if
+ you want a headless host to start the screen by itself on the bot's first
+ `computer_use` call (off by default: installing TigerVNC never yields a screen
+ nobody asked for). A headed browser opens on the screen once it is running.
+2. The pane streams the bot's desktop. The chip in the header says who is in
+ control: **Bot is in control** by default.
+3. Click **Take over**. The border turns red, your keyboard and mouse now drive
+ the bot's screen. Sign in, solve the CAPTCHA, approve the payment.
+4. Click **Hand back**. The bot regains control and re-captures the screen
+ before continuing. Closing the pane also hands control back. A *dropped*
+ connection is different: if your laptop lid closes or Wi-Fi drops while you
+ hold control, you keep it — the bot stays locked out of a screen you may be
+ mid-login on — until you reconnect and hand back. If you come back after a
+ reload and the pane still says a human holds control, a **Hand back (force)**
+ button appears to clear it.
+
+While you hold control, the bot's `computer_use` and browser tools are refused
+with `human_has_control`, captures included. This is a tool-level fence, not an
+OS one: the bot runs as the same user as its screen. Don't type secrets into a
+bot you wouldn't trust with them.
+
+The bot can ask for you: when it recognises a login or verification step it
+calls `computer_use` with `action: "request_handoff"` and a reason, the pane
+shows **Bot needs you**, the bot tells you in its reply what it needs (so the
+ask reaches you in whatever chat you are on), and it blocks in
+`action: "wait_for_human"` until you hand back.
+
+Two viewers on one screen: the most recent **Take over** wins; the previous
+controller drops back to watching.
+
+## Browser sessions that survive the handoff
+
+While the screen runs, the bot's browser tool and the dock's **Browser** icon are
+the same browser: the Chromium agent-browser drives, with one persistent
+user-data-dir per bot (`/bot-desktop/browser-profile`; set
+`AGENT_BROWSER_PROFILE` to pin your own). Click Browser during a takeover and you
+are in the bot's own windows and cookie jar; what you sign in to is what the bot
+uses afterwards and in every later session, until the site expires the login.
+Set `browser.headed: true` so the bot's own browsing is visible on the screen too.
+
+## CLI
+
+```bash
+hermes computer-use screen status # installed? running? who holds control?
+hermes computer-use screen start # start this profile's screen
+hermes computer-use screen stop # stop it (hands control back first)
+hermes computer-use screen install [-y] # apt/dnf/pacman the packages
+hermes -p research computer-use screen start # another bot's screen
+```
+
+## Configuration
+
+```yaml
+bot_desktop:
+ geometry: "1440x900" # screen size; the viewer scales to fit the pane
+ auto_start: true # start on the first computer_use call when the host has no display
+```
+
+State lives under `/bot-desktop/` per profile (RFB Unix socket,
+Xauthority, launcher log, per-profile xfconf).
+
+## How it works
+
+- **TigerVNC `Xvnc`** is the X server and the RFB server in one process, per
+ profile, listening only on a `0600` Unix socket. No TCP port, no VNC
+ password: only processes running as the gateway's user can reach it (see the
+ threat model above), and the gateway's WebSocket bridge is the authenticated
+ way in.
+- **Xfce** starts component-wise (`xfsettingsd`, `xfwm4 --compositor=off`,
+ `xfdesktop`, `xfce4-panel`) under a private D-Bus session, without
+ `xfce4-session`, so nothing tries to lock the screen or reach `logind`.
+- **Hermes Desktop** bundles noVNC. It asks the gateway for a single-use ticket
+ (`display.observe`) over its normal authenticated connection and opens a
+ sibling WebSocket to `/api/display/ws`; the gateway splices the RFB stream
+ through. Nothing new is exposed; the pane works over local, SSH, URL+token
+ and Hermes Cloud connections alike.
+- **Control lease.** The gateway drops keyboard, pointer and clipboard messages
+ from any viewer that does not hold the lease, at the RFB byte level; noVNC's
+ view-only flag is only the UI hint. The same lease gates `computer_use` and
+ the browser tools. It is a file under `/bot-desktop/`: no file
+ means the bot holds control (a fresh profile); a file that exists but cannot
+ be read or parsed fails closed — the bot is treated as locked out until the
+ next successful hand-off rewrites it. Xvnc never pushes the screen's clipboard
+ to viewers (`-SendCutText=0`), so watchers do not receive what the person in
+ control copies; pasting into the screen still works.
+- **Display binding.** The launcher publishes `DISPLAY`, `XAUTHORITY` and the
+ D-Bus address; every cua-driver and headed-browser spawn for that profile
+ inherits them, so the bot never acts on a display a human is sitting at.
+
+## Troubleshooting
+
+- **"Screen packages missing"** — click **Install on host** in the pane, or run
+ the printed install line on the gateway host (not on the machine running
+ Hermes Desktop). The pane refuses a second install while one is running.
+- **Screen starts then stops** — read `/bot-desktop/launcher.log`.
+- **Typing produces wrong characters** — the screen uses a US keymap so RFB
+ keysyms and cua-driver agree; change it with `setxkbmap` on that `DISPLAY`
+ if you need another layout.
+- **Bot says `human_has_control` after you left** — click **Hand back** in the
+ pane (or **Hand back (force)** after a reload). From a shell,
+ `hermes computer-use screen stop` releases the lease and stops the screen;
+ `hermes computer-use screen start` brings it back with the bot in control.
diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md
index 1148e2949a008..3bb53d2d20958 100644
--- a/website/docs/user-guide/features/computer-use.md
+++ b/website/docs/user-guide/features/computer-use.md
@@ -410,10 +410,11 @@ of screenshot context, not ~600K.
[windows-ssh](https://cua.ai/docs/how-to-guides/driver/windows-ssh)
has the recipe.
- **Linux** requires a reachable display server. Headless servers
- need Xvfb (`Xvfb :99 -screen 0 1920x1080x24`) before
- `computer_use` can capture or inject events. Pure Wayland sessions
- need an XWayland bridge for screen capture (cua-driver's Wayland
- inject path handles input independently).
+ get one from [Bot Screen](./bot-screen.md): a per-profile Xfce
+ desktop over TigerVNC that Hermes starts on first use and streams
+ into Hermes Desktop, where you can take over for logins and 2FA.
+ Pure Wayland sessions need an XWayland bridge for screen capture
+ (cua-driver's Wayland inject path handles input independently).
For cross-platform GUI automation without the desktop overhead (and
without TCC / Session 0 / X11 setup), the `browser` toolset uses a
diff --git a/website/sidebars.ts b/website/sidebars.ts
index 55d1ed80160a0..621205071ac50 100644
--- a/website/sidebars.ts
+++ b/website/sidebars.ts
@@ -121,6 +121,7 @@ const sidebars: SidebarsConfig = {
'user-guide/features/browser',
'user-guide/features/credential-vault',
'user-guide/features/computer-use',
+ 'user-guide/features/bot-screen',
'user-guide/features/vision',
'user-guide/features/image-generation',
'user-guide/features/spotify',