diff --git a/apps/desktop/package.json b/apps/desktop/package.json index b21be619de875..b9956ef30f1ad 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -96,6 +96,7 @@ "@icons-pack/react-simple-icons": "13.11.1", "@lezer/highlight": "1.2.3", "@nanostores/react": "1.1.0", + "@novnc/novnc": "1.7.0", "@nous-research/ui": "0.18.2", "@streamdown/code": "1.1.1", "@streamdown/math": "1.0.2", diff --git a/apps/desktop/src/app/chat/close-tab.test.ts b/apps/desktop/src/app/chat/close-tab.test.ts index 219f20df6d495..f0785a6bc9828 100644 --- a/apps/desktop/src/app/chat/close-tab.test.ts +++ b/apps/desktop/src/app/chat/close-tab.test.ts @@ -7,6 +7,12 @@ const nextSessionTileForWorkspace = vi.fn<() => null | string>(() => null) const closeSessionTile = vi.fn() const requestFreshSession = vi.fn() +const closeActiveTerminal = vi.fn() + +vi.mock('@/app/right-sidebar/terminal/terminals', () => ({ + closeActiveTerminal: () => closeActiveTerminal() +})) + vi.mock('@/components/pane-shell/tree/store', () => ({ closeFocusedSessionTab: () => closeFocusedSessionTab(), closeFocusedToolTab: () => closeFocusedToolTab() @@ -136,6 +142,20 @@ describe('closeWorkspaceTab', () => { expect(requestFreshSession).toHaveBeenCalledTimes(1) }) + it('a focused remote bot screen swallows ⌘W: no terminal tab, no session tab closes', async () => { + loadedMainOnly() + const combo = await import('@/lib/keybinds/combo') + const spy = vi.spyOn(combo, 'isFocusWithin').mockImplementation(selector => selector === '[data-remote-screen]' || selector === '[data-terminal]') + + try { + expect(closeActiveTab(vi.fn())).toBe(true) + expect(closeActiveTerminal).not.toHaveBeenCalled() + expect(requestFreshSession).not.toHaveBeenCalled() + } finally { + spy.mockRestore() + } + }) + it('a focused tool panel (terminal / logs) claims ⌘W before main empties', () => { loadedMainOnly() closeFocusedToolTab.mockReturnValue(true) diff --git a/apps/desktop/src/app/chat/close-tab.ts b/apps/desktop/src/app/chat/close-tab.ts index 4304138904ff5..6aa681fba2274 100644 --- a/apps/desktop/src/app/chat/close-tab.ts +++ b/apps/desktop/src/app/chat/close-tab.ts @@ -67,6 +67,12 @@ export function closeWorkspaceTab(loadSessionIntoWorkspace?: (storedSessionId: s * with its own tab strip closes ITS tab instead of main's. */ export function closeActiveTab(loadSessionIntoWorkspace?: (storedSessionId: string) => void): boolean { + // A remote bot screen borrows the terminal's keyboard ownership marker so bare keys reach it; ⌘W + // there belongs to the remote desktop, never to a local terminal tab or the session tab behind it. + if (isFocusWithin('[data-remote-screen]')) { + return true + } + if (isFocusWithin('[data-terminal]')) { closeActiveTerminal() diff --git a/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx b/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx index 82281dcd4126a..829c85ff0815c 100644 --- a/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx +++ b/apps/desktop/src/app/chat/sidebar/gateway-groups.tsx @@ -2,9 +2,10 @@ import type { useSensors } from '@dnd-kit/core' import { arrayMove } from '@dnd-kit/sortable' import { useStore } from '@nanostores/react' import type { ReactNode } from 'react' -import { useState } from 'react' +import { useMemo, useState } from 'react' import { type NewSessionSplitHandler, startNewSessionDrag } from '@/app/chat/new-session-drag' +import { type ProfileGroupHeaderContribution, SIDEBAR_PROFILE_GROUP_HEADER_AREA } from '@/app/routes' import { Button } from '@/components/ui/button' import { Codicon } from '@/components/ui/codicon' import { @@ -18,6 +19,8 @@ import { import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu' import { Input } from '@/components/ui/input' import { ProfileGlyph } from '@/components/ui/profile-glyph' +import { useContributions } from '@/contrib' +import { ContribBoundary, ContribRender } from '@/contrib/react/boundary' import type { SessionInfo } from '@/hermes' import { useI18n } from '@/i18n' import { useStoreSelector } from '@/lib/use-session-slice' @@ -272,6 +275,7 @@ function GatewayProfileGroup({ {open && ( <> {children} + {group.profile ? : null} {renderRows(sessions.slice(0, visibleCount))} {hiddenCount > 0 && ( ) } + +/** Plugin-contributed chrome at the top of one expanded gateway/profile group + * (`sidebar.profileGroup.header`): the Bots plugin mounts its Screen portal + * here so the profile's computer is one click away from its sessions. */ +function ProfileGroupHeaderSlot({ connectionId, profile }: { connectionId: null | string; profile: string }) { + const items = useContributions(SIDEBAR_PROFILE_GROUP_HEADER_AREA) + + if (!items.length) { + return null + } + + return ( +
+ {items.map(item => { + const data = item.data as Partial | undefined + + if (typeof data?.render !== 'function') { + return null + } + + return ( + + + + ) + })} +
+ ) +} + +/** One stable render identity per (render, connection, profile): ContribRender mounts whatever + * function it is handed, so an inline closure would remount the contribution on every paint. */ +function ProfileGroupHeaderItem({ + connectionId, + profile, + render +}: { + connectionId: null | string + profile: string + render: ProfileGroupHeaderContribution['render'] +}) { + const Row = useMemo(() => () => render({ connectionId, profile }), [connectionId, profile, render]) + + return +} diff --git a/apps/desktop/src/app/routes.ts b/apps/desktop/src/app/routes.ts index 7e5fbd4718263..c3a08f529899d 100644 --- a/apps/desktop/src/app/routes.ts +++ b/apps/desktop/src/app/routes.ts @@ -123,6 +123,25 @@ export interface SidebarNavContribution { path: string } +// ── Contributed profile-group header — the `sidebar.profileGroup.header` area ─ +// A RENDER contribution mounted at the top of each gateway/profile group in the +// Sessions sidebar (above its session rows) while the group is expanded. The +// contribution's `data` is a `ProfileGroupHeaderContribution`; core calls +// `render(route)` with the group's connection + profile so one contribution +// serves every group. First consumer: the Bots plugin's Screen portal. + +export const SIDEBAR_PROFILE_GROUP_HEADER_AREA = 'sidebar.profileGroup.header' + +export interface ProfileGroupRoute { + connectionId: null | string + profile: string +} + +/** Payload of a `sidebar.profileGroup.header` data contribution. */ +export interface ProfileGroupHeaderContribution { + render: (route: ProfileGroupRoute) => ReactNode +} + // Views that render as a full-screen modal card (OverlayView) over the shell. // While one is open the app's titlebar control clusters must hide so they don't // bleed over the overlay (they sit at a higher z-index than the overlay card). diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts index 9319993797cab..2ad74121fb2c5 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts @@ -12,10 +12,12 @@ import { import { $gateway } from '@/store/gateway' import { setMcpSetupRequest } from '@/store/mcp-setup' import { dispatchNativeNotification } from '@/store/native-notifications' +import { $activeGatewayProfile } from '@/store/profile' import { $vaultCodeRequests, $vaultSaveLoginRequests, $vaultUnlockRequests, + clearSudoRequest, clearVaultCodeRequest, clearVaultSaveLoginRequest, clearVaultUnlockRequest, @@ -203,6 +205,14 @@ export function handleInputRequestEvent(ctx: GatewayEventContext): boolean { return true } + if (event.type === 'sudo.expire' || event.type === 'display.install.sudo.expire') { + // The backend gave up waiting; tear the card down so a late Send cannot go anywhere. + const requestId = typeof payload?.request_id === 'string' ? payload.request_id : '' + clearSudoRequest(sessionId ?? undefined, requestId || undefined) + + return true + } + if (event.type === 'clarify.expire') { if (!sessionId) { return true @@ -313,13 +323,20 @@ export function handleInputRequestEvent(ctx: GatewayEventContext): boolean { return true } - if (event.type === 'sudo.request') { - // Sudo password capture (tools/terminal_tool.py). Blocked on - // sudo.respond {request_id, password}. + if (event.type === 'sudo.request' || event.type === 'display.install.sudo.request') { + // Sudo password capture (tools/terminal_tool.py), or the Bot Screen package install + // (tui_gateway/methods_display.py) reusing the same masked card. Blocked on + // .respond {request_id, password}. const requestId = typeof payload?.request_id === 'string' ? payload.request_id : '' + const install = event.type === 'display.install.sudo.request' if (requestId) { - setSudoRequest({ requestId, sessionId: sessionId ?? null }) + setSudoRequest({ + requestId, + sessionId: sessionId ?? null, + origin: { connectionId: event.connectionId ?? null, profile: event.profile ?? $activeGatewayProfile.get() }, + ...(install ? { respondMethod: 'display.install.sudo.respond', description: translateNow('prompts.sudoInstallDesc') } : {}) + }) if (sessionId) { updateSessionState(sessionId, state => ({ ...state, needsInput: true })) diff --git a/apps/desktop/src/components/prompt-overlays.tsx b/apps/desktop/src/components/prompt-overlays.tsx index 091e306ad5c7f..6c6c5647c2c18 100644 --- a/apps/desktop/src/components/prompt-overlays.tsx +++ b/apps/desktop/src/components/prompt-overlays.tsx @@ -19,7 +19,7 @@ import { useI18n } from '@/i18n' import { isMissingPendingPromptRequest } from '@/lib/gateway-rpc' import { triggerHaptic } from '@/lib/haptics' import { KeyRound, Loader2, Lock, ShieldLock } from '@/lib/icons' -import { $gateway } from '@/store/gateway' +import { $gateway, requestGatewayForAgent } from '@/store/gateway' import { notifyError } from '@/store/notifications' import { clearSecretRequest, @@ -78,10 +78,16 @@ function SudoDialog({ sessionId }: { sessionId: string | null }) { setSubmitting(true) try { - await gateway.request<{ status?: string }>('sudo.respond', { - password: value, - request_id: request.requestId - }) + const method = request.respondMethod ?? 'sudo.respond' + const reply = { password: value, request_id: request.requestId } + + // Pinned to the socket the request came from: the foreground gateway may be another host. + if (request.origin) { + await requestGatewayForAgent<{ status?: string }>(request.origin.connectionId, request.origin.profile, method, reply) + } else { + await gateway.request<{ status?: string }>(method, reply) + } + triggerHaptic('submit') clearSudoRequest(request.sessionId, request.requestId) } catch (error) { @@ -126,7 +132,7 @@ function SudoDialog({ sessionId }: { sessionId: string | null }) { {copy.sudoTitle} - {copy.sudoDesc} + {request.description ?? copy.sudoDesc}
diff --git a/apps/desktop/src/i18n/ar.ts b/apps/desktop/src/i18n/ar.ts index 6d70110018d12..47c44c9ebb39c 100644 --- a/apps/desktop/src/i18n/ar.ts +++ b/apps/desktop/src/i18n/ar.ts @@ -3046,6 +3046,7 @@ export const ar = defineLocale({ secretSendFailed: 'فشل إرسال السر', sudoTitle: 'مطلوب sudo', sudoDesc: 'أدخل كلمة المرور لمتابعة الأمر.', + sudoInstallDesc: 'يحتاج Hermes إلى كلمة مرور sudo لتثبيت حزم Bot Screen (TigerVNC + Xfce) على مضيف البوابة. تُرسل إلى ذلك المضيف فقط.', sudoPlaceholder: 'كلمة المرور', secretTitle: 'مطلوب سر', secretDesc: 'أدخل القيمة المطلوبة لمتابعة المهمة.', diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index acbde2d1142b4..a5500c5e4775c 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -4011,6 +4011,7 @@ export const en: Translations = { secretSendFailed: 'Could not send secret', sudoTitle: 'Administrator password', sudoDesc: 'Hermes needs your sudo password to run a privileged command. It is sent only to your local agent.', + sudoInstallDesc: 'Hermes needs your sudo password to install the Bot Screen packages (TigerVNC + Xfce) on the gateway host. It is sent only to that host.', sudoPlaceholder: 'sudo password', secretTitle: 'Secret required', secretDesc: 'Hermes needs a credential to continue.', diff --git a/apps/desktop/src/i18n/ja.ts b/apps/desktop/src/i18n/ja.ts index cdd512551c88d..e870777c436af 100644 --- a/apps/desktop/src/i18n/ja.ts +++ b/apps/desktop/src/i18n/ja.ts @@ -3456,6 +3456,7 @@ export const ja = defineLocale({ sudoTitle: '管理者パスワード', sudoDesc: 'Hermes は特権コマンドを実行するために sudo パスワードが必要です。ローカルエージェントにのみ送信されます。', + sudoInstallDesc: 'Bot Screen のパッケージ(TigerVNC + Xfce)をゲートウェイホストにインストールするため、sudo パスワードが必要です。そのホストにのみ送信されます。', sudoPlaceholder: 'sudo パスワード', secretTitle: 'シークレットが必要です', secretDesc: 'Hermes は続行するための認証情報が必要です。', diff --git a/apps/desktop/src/i18n/ru.ts b/apps/desktop/src/i18n/ru.ts index e8ffa262a36c3..5f7a04d42b254 100644 --- a/apps/desktop/src/i18n/ru.ts +++ b/apps/desktop/src/i18n/ru.ts @@ -3753,6 +3753,7 @@ export const ru = defineLocale({ sudoTitle: 'Пароль администратора', sudoDesc: 'Hermes нужен ваш пароль sudo, чтобы выполнить команду с повышенными правами. Он отправляется только вашему локальному агенту.', + sudoInstallDesc: 'Hermes нужен ваш пароль sudo, чтобы установить пакеты Bot Screen (TigerVNC + Xfce) на хосте шлюза. Он отправляется только на этот хост.', sudoPlaceholder: 'пароль sudo', secretTitle: 'Требуется секрет', secretDesc: 'Hermes нужны учётные данные, чтобы продолжить.', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 0c16cb4b069ac..e8b4a3eb25b7a 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -3505,6 +3505,7 @@ export interface Translations { secretSendFailed: string sudoTitle: string sudoDesc: string + sudoInstallDesc: string sudoPlaceholder: string secretTitle: string secretDesc: string diff --git a/apps/desktop/src/i18n/zh-hant.ts b/apps/desktop/src/i18n/zh-hant.ts index 4759253d154e9..21bc27f7fa618 100644 --- a/apps/desktop/src/i18n/zh-hant.ts +++ b/apps/desktop/src/i18n/zh-hant.ts @@ -3311,6 +3311,7 @@ export const zhHant = defineLocale({ secretSendFailed: '無法傳送密鑰', sudoTitle: '管理員密碼', sudoDesc: 'Hermes 需要您的 sudo 密碼來執行特權指令。它只會傳送給您的本機代理。', + sudoInstallDesc: 'Hermes 需要您的 sudo 密碼,以在閘道主機上安裝 Bot Screen 套件(TigerVNC + Xfce)。它只會傳送到該主機。', sudoPlaceholder: 'sudo 密碼', secretTitle: '需要密鑰', secretDesc: 'Hermes 需要一個憑證才能繼續。', diff --git a/apps/desktop/src/i18n/zh.ts b/apps/desktop/src/i18n/zh.ts index 7e78217d1b7cd..26f00c698daab 100644 --- a/apps/desktop/src/i18n/zh.ts +++ b/apps/desktop/src/i18n/zh.ts @@ -4124,6 +4124,7 @@ export const zh = defineLocale({ secretSendFailed: '无法发送密钥', sudoTitle: '管理员密码', sudoDesc: 'Hermes 需要你的 sudo 密码来运行特权命令。它只会发送给你的本地 agent。', + sudoInstallDesc: 'Hermes 需要你的 sudo 密码,以在网关主机上安装 Bot Screen 软件包(TigerVNC + Xfce)。它只会发送到该主机。', sudoPlaceholder: 'sudo 密码', secretTitle: '需要密钥', secretDesc: 'Hermes 需要一个凭据才能继续。', diff --git a/apps/desktop/src/lib/sibling-ws-url.test.ts b/apps/desktop/src/lib/sibling-ws-url.test.ts new file mode 100644 index 0000000000000..06ac27e43edfa --- /dev/null +++ b/apps/desktop/src/lib/sibling-ws-url.test.ts @@ -0,0 +1,49 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { resolveSiblingWsUrl } from './sibling-ws-url' + +// A sibling stream (voice PCM, Bot Screen RFB) must dial the SAME (connection, +// profile) backend chat uses. The bare v1 getConnection pair answers for the +// local primary — the wrong machine when a registry remote rides over a local +// install — so registry routes must go through the *For bridges. +describe('resolveSiblingWsUrl', () => { + const remoteWsUrl = 'wss://gateway.example/api/ws?ticket=fresh' + const localWsUrl = 'ws://127.0.0.1:5151/api/ws?token=local' + + let getConnection: ReturnType + let getConnectionFor: ReturnType + let getGatewayWsUrl: ReturnType + let getGatewayWsUrlFor: ReturnType + + beforeEach(() => { + getConnection = vi.fn(async () => ({ authMode: 'token', baseUrl: 'http://127.0.0.1:5151', wsUrl: localWsUrl })) + getConnectionFor = vi.fn(async () => ({ authMode: 'token', baseUrl: 'https://gateway.example', wsUrl: remoteWsUrl })) + getGatewayWsUrl = vi.fn(async () => ({ ok: true, wsUrl: localWsUrl })) + getGatewayWsUrlFor = vi.fn(async () => ({ ok: true, wsUrl: remoteWsUrl })) + Object.defineProperty(window, 'hermesDesktop', { + configurable: true, + value: { getConnection, getConnectionFor, getGatewayWsUrl, getGatewayWsUrlFor } + }) + }) + + afterEach(() => { + Reflect.deleteProperty(window, 'hermesDesktop') + }) + + it('routes a registry-scoped profile through the *For bridges and swaps only the path', async () => { + const url = await resolveSiblingWsUrl({ connectionId: 'gw-tailscale', profile: 'research' }, '/api/display/ws') + + expect(url).toBe('wss://gateway.example/api/display/ws?ticket=fresh') + expect(getConnectionFor).toHaveBeenCalledWith({ connectionId: 'gw-tailscale', profile: 'research' }) + expect(getConnection).not.toHaveBeenCalled() + expect(getGatewayWsUrl).not.toHaveBeenCalled() + }) + + it('strips the spent gateway credential when the sibling route authenticates itself', async () => { + const url = new URL(await resolveSiblingWsUrl({ profile: null }, 'api/display/ws', { stripGatewayCredential: true })) + + expect(url.origin + url.pathname).toBe('ws://127.0.0.1:5151/api/display/ws') + expect(url.searchParams.has('token')).toBe(false) + expect(url.searchParams.has('ticket')).toBe(false) + }) +}) diff --git a/apps/desktop/src/lib/sibling-ws-url.ts b/apps/desktop/src/lib/sibling-ws-url.ts new file mode 100644 index 0000000000000..7ba0be8045be4 --- /dev/null +++ b/apps/desktop/src/lib/sibling-ws-url.ts @@ -0,0 +1,86 @@ +/** + * Sibling WebSocket URLs beside `/api/ws` for a (connection, profile) route. + * + * Some gateway streams are not JSON-RPC and cannot share the gateway socket: + * voice PCM (`/api/audio/speak-stream`) and the Bot Screen's raw RFB + * (`/api/display/ws`). They ride the SAME authenticated origin the route's + * `/api/ws` uses — a fresh credential for OAuth remotes, the registry-scoped + * `*For` bridges for a remote riding over a local install (the bare + * getConnection/getGatewayWsUrl pair answers for the v1 primary backend, which + * would be the wrong machine). One resolver so every sibling stream routes the + * way chat does. + */ + +import { resolveGatewayWsUrl } from '@hermes/shared' + +const RESOLVE_TIMEOUT_MS = 15_000 + +function withTimeout(promise: Promise, ms: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = window.setTimeout(() => reject(new Error(label)), ms) + promise.then( + value => { + window.clearTimeout(timer) + resolve(value) + }, + error => { + window.clearTimeout(timer) + reject(error instanceof Error ? error : new Error(String(error))) + } + ) + }) +} + +export interface SiblingWsRoute { + connectionId?: null | string + profile?: null | string +} + +/** + * Resolve `ws(s):///` for `route`. `stripGatewayCredential` + * drops the `?ticket=`/`?token=` the gateway URL carried when the sibling route + * authenticates on its own credential (a one-shot ticket must not be spent + * twice; the display bridge mints its own). + */ +export async function resolveSiblingWsUrl( + route: SiblingWsRoute, + path: string, + options: { stripGatewayCredential?: boolean } = {} +): Promise { + const desktop = window.hermesDesktop + + if (!desktop?.getConnection) { + throw new Error('Hermes Desktop connection bridge unavailable') + } + + const connectionId = route.connectionId?.trim() || null + const profile = route.profile?.trim() || null + + const conn = + connectionId && desktop.getConnectionFor + ? await withTimeout(desktop.getConnectionFor({ connectionId, profile }), RESOLVE_TIMEOUT_MS, `Timed out connecting to profile "${profile}"`) + : await withTimeout(desktop.getConnection(profile), RESOLVE_TIMEOUT_MS, `Timed out connecting to profile "${profile}"`) + + const wsDeps = + connectionId && desktop.getGatewayWsUrlFor + ? { getGatewayWsUrl: () => desktop.getGatewayWsUrlFor!({ connectionId, profile }) } + : connectionId + ? {} + : desktop + + const wsUrl = await withTimeout(resolveGatewayWsUrl(wsDeps, conn), RESOLVE_TIMEOUT_MS, 'Timed out minting the gateway WebSocket URL') + const url = new URL(wsUrl) + + if (!url.pathname.endsWith('/api/ws')) { + throw new Error(`Unexpected gateway WebSocket path: ${url.pathname}`) + } + + url.pathname = url.pathname.replace(/\/api\/ws$/, path.startsWith('/') ? path : `/${path}`) + + if (options.stripGatewayCredential) { + url.searchParams.delete('ticket') + url.searchParams.delete('token') + } + + return url.toString() +} diff --git a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx index f21a1210264d0..ab4ed243729c3 100644 --- a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx +++ b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx @@ -72,6 +72,7 @@ import { useTurnBusy, workerActiveAt } from './row-helpers' +import { openBotScreen } from './screen-open' import type { GroupMember, RosterRow, SidebarRowLabels } from './types' import { $botSections, $draggingBot, BOT_DRAG_MIME, botSectionId, moveBotsToSection } from './user-sections' @@ -310,6 +311,7 @@ export function BotRow({ bot, onDelete, onEdit, onGroup, onNewSection, showHandl {row} void openRosterBot(bot)}>{b.bot.openBotChat} + openBotScreen(bot, meta)}>{b.screen.menu} { diff --git a/apps/desktop/src/plugins/hermes-bots/cron.tsx b/apps/desktop/src/plugins/hermes-bots/cron.tsx index 931bd069ac609..c786f276d8f0f 100644 --- a/apps/desktop/src/plugins/hermes-bots/cron.tsx +++ b/apps/desktop/src/plugins/hermes-bots/cron.tsx @@ -45,6 +45,7 @@ import { labeled } from './dialog-parts' import { botsText, useBots } from './i18n' import { displayName } from './labels' import { botConnectionRoute, botRosterMeta, requestForBot } from './routing' +import { ScreenHero } from './screen-hero' import { ID } from './shared' import type { BotMeta, RosterRow, RoutineJob } from './types' @@ -1240,6 +1241,9 @@ export function RoutinesPane() { return (
+
+ +
diff --git a/apps/desktop/src/plugins/hermes-bots/i18n.ts b/apps/desktop/src/plugins/hermes-bots/i18n.ts index d6b96cf348bc1..fd860551e2d59 100644 --- a/apps/desktop/src/plugins/hermes-bots/i18n.ts +++ b/apps/desktop/src/plugins/hermes-bots/i18n.ts @@ -223,6 +223,56 @@ type BotsMessages = { noMcpServers: string } + /** Bot Screen: the bot's headless desktop on the gateway host, live in a pane. */ + screen: { + title: string + menu: string + unsupportedTitle: string + unsupportedBody: string + notInstalledTitle: string + notInstalledBody: string + installHint: string + install: string + installing: string + installCancelled: string + installFailed: string + noPackageManager: string + portalTitle: string + portalOpen: string + heroStopped: string + heroNotInstalled: string + heroConnecting: string + heroStale: string + heroSuppressed: string + heroOpenLive: string + heroInstall: string + heroStart: string + portalWatching: string + portalYouControl: string + portalOtherControls: string + portalStopped: string + portalNotInstalled: string + portalUnsupported: string + portalUnavailable: string + unavailableTitle: string + stoppedTitle: string + stoppedBody: string + start: string + attaching: string + streamLost: string + reconnect: string + takeOver: string + handBack: string + handBackForce: string + handBackForceHint: string + openNeedsUpdate: string + youControl: string + otherControls: string + agentControls: string + controlTaken: string + handoffRequested: string + } + /** Bot-scoped scheduled jobs. Generic scheduling chrome (weekday names, * Daily/Hourly, the job verbs) resolves against core's `cron` section. */ cron: { @@ -446,6 +496,54 @@ const en: BotsMessages = { searchHub: 'Search the hub (community + well-known sources)…', noMcpServers: 'No MCP servers configured or in the catalog.' }, + screen: { + title: 'Screen', + menu: 'Open Screen', + unsupportedTitle: 'No bot screen on this host', + unsupportedBody: 'Bot screens run on Linux gateway hosts. This bot uses the host\u2019s own display.', + notInstalledTitle: 'Screen packages missing', + notInstalledBody: 'The gateway host needs TigerVNC and the Xfce core to give this bot a screen. Run on the host:', + installHint: 'Runs on the gateway host as the user Hermes runs as; sudo is asked for once, through Hermes.', + install: 'Install on host', + installing: 'Installing…', + installCancelled: 'Install cancelled: no sudo password was provided.', + installFailed: 'Install failed. Read the log above, or run the command on the host yourself.', + noPackageManager: 'No supported package manager (apt, dnf, pacman) was found on the gateway host.', + portalTitle: 'Screen', + portalOpen: 'Open', + heroStopped: 'Screen is off', + heroNotInstalled: 'Not installed on this host', + heroConnecting: 'Checking the screen…', + heroStale: 'Last seen — screen unreachable', + heroSuppressed: 'Hidden while someone has control', + heroOpenLive: 'Open live', + heroInstall: 'Install', + heroStart: 'Start', + portalWatching: 'Live · bot in control', + portalYouControl: 'Live · you are in control', + portalOtherControls: 'Live · another viewer in control', + portalStopped: 'Stopped', + portalNotInstalled: 'Not installed on host', + portalUnsupported: 'Not available on this host', + portalUnavailable: 'Update the bot\u2019s Hermes to use Screen', + unavailableTitle: 'Screen needs a newer Hermes', + stoppedTitle: 'Screen is off', + stoppedBody: 'Start this bot\u2019s desktop to watch what it does and take over when it needs you.', + start: 'Start screen', + attaching: 'Connecting to the screen\u2026', + streamLost: 'Screen stream ended', + reconnect: 'Reconnect', + takeOver: 'Take over', + handBack: 'Hand back', + handBackForce: 'Hand back (force)', + handBackForceHint: 'Release a lease held by a viewer that is no longer here, e.g. after a reload.', + openNeedsUpdate: 'Update Hermes Desktop to open bot screens.', + youControl: 'You are in control', + otherControls: 'Another viewer is in control', + agentControls: 'Bot is in control', + controlTaken: 'Another viewer took control. Watching only.', + handoffRequested: 'Bot needs you' + }, cron: { filterHint: 'Scheduled jobs exist in this profile but none are tagged for this bot. Name a job "[bot:] …" to show it here, or see them in Cron below.', @@ -665,6 +763,54 @@ const ja: BotsMessages = { searchHub: 'ハブを検索(コミュニティと既知のソース)…', noMcpServers: '設定済みまたはカタログ内の MCP サーバーはありません。' }, + screen: { + title: '画面', + menu: '画面を開く', + unsupportedTitle: 'このホストにはボット画面がありません', + unsupportedBody: 'ボット画面は Linux のゲートウェイホストで動作します。このボットはホスト自身のディスプレイを使います。', + notInstalledTitle: '画面パッケージが不足しています', + notInstalledBody: 'このボットに画面を与えるには、ゲートウェイホストに TigerVNC と Xfce コアが必要です。ホストで実行:', + installHint: 'Hermes を実行しているユーザーとしてゲートウェイホスト上で実行されます。sudo は Hermes 経由で一度だけ求められます。', + install: 'ホストにインストール', + installing: 'インストール中…', + installCancelled: 'インストールを中止しました: sudo パスワードが入力されませんでした。', + installFailed: 'インストールに失敗しました。上のログを確認するか、ホストでコマンドを直接実行してください。', + noPackageManager: 'ゲートウェイホストに対応するパッケージマネージャー (apt, dnf, pacman) が見つかりません。', + portalTitle: 'スクリーン', + portalOpen: '開く', + heroStopped: '画面は停止中', + heroNotInstalled: 'このホストには未インストール', + heroConnecting: '画面を確認中…', + heroStale: '最終表示 — 画面に接続できません', + heroSuppressed: '他の人が操作中は非表示', + heroOpenLive: 'ライブで開く', + heroInstall: 'インストール', + heroStart: '開始', + portalWatching: 'ライブ · ボットが操作中', + portalYouControl: 'ライブ · あなたが操作中', + portalOtherControls: 'ライブ · 別のビューアーが操作中', + portalStopped: '停止中', + portalNotInstalled: 'ホストに未インストール', + portalUnsupported: 'このホストでは利用できません', + portalUnavailable: 'Screen を使うにはボットの Hermes を更新してください', + unavailableTitle: 'Screen には新しい Hermes が必要です', + stoppedTitle: '画面はオフです', + stoppedBody: 'このボットのデスクトップを起動すると、動作を見守り、必要なときに操作を引き継げます。', + start: '画面を起動', + attaching: '画面に接続中…', + streamLost: '画面ストリームが終了しました', + reconnect: '再接続', + takeOver: '引き継ぐ', + handBack: '戻す', + handBackForce: '強制的に戻す', + handBackForceHint: 'もう存在しないビューア(再読み込み後など)が保持しているリースを解放します。', + openNeedsUpdate: 'ボットの画面を開くには Hermes Desktop を更新してください。', + youControl: 'あなたが操作中', + otherControls: '別のビューアが操作中', + agentControls: 'ボットが操作中', + controlTaken: '別のビューアが操作を引き継ぎました。閲覧のみ。', + handoffRequested: 'ボットが助けを求めています' + }, cron: { filterHint: 'このプロファイルには定期実行ジョブがありますが、このボット向けのタグが付いたものはありません。ジョブ名を「[bot:<名前>] …」にするとここに表示されます。下のCronでも確認できます。', @@ -879,6 +1025,54 @@ const zh: BotsMessages = { searchHub: '搜索技能中心(社区和常见来源)…', noMcpServers: '未配置 MCP 服务器,目录中也没有。' }, + screen: { + title: '屏幕', + menu: '打开屏幕', + unsupportedTitle: '此主机没有机器人屏幕', + unsupportedBody: '机器人屏幕在 Linux 网关主机上运行。此机器人使用主机自身的显示器。', + notInstalledTitle: '缺少屏幕软件包', + notInstalledBody: '网关主机需要 TigerVNC 和 Xfce 核心组件才能为此机器人提供屏幕。在主机上运行:', + installHint: '在网关主机上以运行 Hermes 的用户身份执行;sudo 只会通过 Hermes 询问一次。', + install: '安装到主机', + installing: '正在安装…', + installCancelled: '安装已取消:未提供 sudo 密码。', + installFailed: '安装失败。请查看上方日志,或在主机上手动运行该命令。', + noPackageManager: '网关主机上未找到受支持的包管理器(apt、dnf、pacman)。', + portalTitle: '屏幕', + portalOpen: '打开', + heroStopped: '屏幕已关闭', + heroNotInstalled: '此主机未安装', + heroConnecting: '正在检查屏幕…', + heroStale: '最后画面 — 屏幕无法访问', + heroSuppressed: '有人控制时隐藏', + heroOpenLive: '实时打开', + heroInstall: '安装', + heroStart: '启动', + portalWatching: '直播 · 机器人控制中', + portalYouControl: '直播 · 你在控制', + portalOtherControls: '直播 · 其他查看者控制中', + portalStopped: '已停止', + portalNotInstalled: '主机未安装', + portalUnsupported: '此主机不可用', + portalUnavailable: '更新机器人的 Hermes 以使用屏幕', + unavailableTitle: '屏幕需要更新版的 Hermes', + stoppedTitle: '屏幕已关闭', + stoppedBody: '启动此机器人的桌面,观看它的操作,并在需要时接管。', + start: '启动屏幕', + attaching: '正在连接屏幕…', + streamLost: '屏幕流已结束', + reconnect: '重新连接', + takeOver: '接管', + handBack: '交还', + handBackForce: '强制交还', + handBackForceHint: '释放已不在场的查看者(例如重新加载后)持有的控制权。', + openNeedsUpdate: '更新 Hermes Desktop 以打开机器人屏幕。', + youControl: '你正在控制', + otherControls: '另一位查看者正在控制', + agentControls: '机器人正在控制', + controlTaken: '另一位查看者已接管控制。仅可观看。', + handoffRequested: '机器人需要你' + }, cron: { filterHint: '此配置档案中有定时任务,但没有一个标记给这个机器人。将任务命名为“[bot:<名称>] …”即可显示在这里,也可以在下方的 Cron 中查看。', @@ -1093,6 +1287,54 @@ const zhHant: BotsMessages = { searchHub: '搜尋技能中心(社群和常見來源)…', noMcpServers: '未設定 MCP 伺服器,目錄中也沒有。' }, + screen: { + title: '螢幕', + menu: '開啟螢幕', + unsupportedTitle: '此主機沒有機器人螢幕', + unsupportedBody: '機器人螢幕在 Linux 閘道主機上執行。此機器人使用主機自身的顯示器。', + notInstalledTitle: '缺少螢幕套件', + notInstalledBody: '閘道主機需要 TigerVNC 與 Xfce 核心元件才能為此機器人提供螢幕。在主機上執行:', + installHint: '在閘道主機上以執行 Hermes 的使用者身分執行;sudo 只會透過 Hermes 詢問一次。', + install: '安裝到主機', + installing: '安裝中…', + installCancelled: '安裝已取消:未提供 sudo 密碼。', + installFailed: '安裝失敗。請查看上方日誌,或在主機上手動執行該指令。', + noPackageManager: '閘道主機上找不到受支援的套件管理器(apt、dnf、pacman)。', + portalTitle: '螢幕', + portalOpen: '開啟', + heroStopped: '螢幕已關閉', + heroNotInstalled: '此主機未安裝', + heroConnecting: '正在檢查螢幕…', + heroStale: '最後畫面 — 螢幕無法連線', + heroSuppressed: '有人控制時隱藏', + heroOpenLive: '即時開啟', + heroInstall: '安裝', + heroStart: '啟動', + portalWatching: '直播 · 機器人控制中', + portalYouControl: '直播 · 您在控制', + portalOtherControls: '直播 · 其他檢視者控制中', + portalStopped: '已停止', + portalNotInstalled: '主機未安裝', + portalUnsupported: '此主機不可用', + portalUnavailable: '更新機器人的 Hermes 以使用螢幕', + unavailableTitle: '螢幕需要較新版的 Hermes', + stoppedTitle: '螢幕已關閉', + stoppedBody: '啟動此機器人的桌面,觀看它的操作,並在需要時接手。', + start: '啟動螢幕', + attaching: '正在連線至螢幕…', + streamLost: '螢幕串流已結束', + reconnect: '重新連線', + takeOver: '接手', + handBack: '交還', + handBackForce: '強制交還', + handBackForceHint: '釋放已不在場的檢視者(例如重新載入後)持有的控制權。', + openNeedsUpdate: '更新 Hermes Desktop 以開啟機器人螢幕。', + youControl: '你正在控制', + otherControls: '另一位檢視者正在控制', + agentControls: '機器人正在控制', + controlTaken: '另一位檢視者已接手控制。僅可觀看。', + handoffRequested: '機器人需要你' + }, cron: { filterHint: '此設定檔中有排程工作,但沒有任何一個標記給這個機器人。將工作命名為「[bot:<名稱>] …」即可顯示在這裡,也可以在下方的 Cron 中查看。', diff --git a/apps/desktop/src/plugins/hermes-bots/plugin.tsx b/apps/desktop/src/plugins/hermes-bots/plugin.tsx index 8032e16134ffa..d472473b6dff9 100644 --- a/apps/desktop/src/plugins/hermes-bots/plugin.tsx +++ b/apps/desktop/src/plugins/hermes-bots/plugin.tsx @@ -15,8 +15,8 @@ * bot-initiated sends use `hermes -p chat --in ~ -c "Bot Chat"`. */ -import { CHAT_EMPTY_AREA, COMPOSER_AREAS, host, PALETTE_AREA, translateNow } from '@hermes/plugin-sdk' -import type { ChatEmptyProps, PluginContext } from '@hermes/plugin-sdk' +import { CHAT_EMPTY_AREA, COMPOSER_AREAS, host, PALETTE_AREA, SIDEBAR_PROFILE_GROUP_HEADER_AREA, translateNow } from '@hermes/plugin-sdk' +import type { ChatEmptyProps, PluginContext, ProfileGroupRoute } from '@hermes/plugin-sdk' import { startFaceClock, stopFaceClock } from './avatar' import { @@ -69,6 +69,7 @@ import { sessionOwnsWorkspace } from './roster-pane' import { botRosterMeta, botWorkspaceOwnerKey, setBotsWorkspaceOwner } from './routing' +import { ProfileGroupScreenPortal } from './screen-portal' import { startHideSweepScheduler } from './session-sweep' import { bumpBotOpenGeneration, getBotOpenGeneration, ID, setPluginCtx } from './shared' import type { GroupChat, RosterRow } from './types' @@ -363,6 +364,13 @@ export default { // the meta/room storage hydrates above have landed; idempotent after that. // (Feature-guarded: bare vm test harnesses have no setTimeout global.) startHideSweepScheduler(ctx) + // Sessions sidebar: each gateway/profile group gets the profile's Screen portal + // above its sessions, so the bot's computer is reachable from either mode. + ctx.register({ + id: 'screen-portal', + area: SIDEBAR_PROFILE_GROUP_HEADER_AREA, + data: { render: (route: ProfileGroupRoute) => } + }) ctx.register({ id: 'pane', area: 'panes', diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts new file mode 100644 index 0000000000000..f4f36759bf730 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.test.ts @@ -0,0 +1,44 @@ +/** + * Two remote hosts can share the same `~/.hermes` path, so a `display.*` event + * matched on `profile_key` alone would let host B's take-over repaint host A's + * screen pane. The event must also have arrived on the bot's own connection. + */ + +import { describe, expect, it, vi } from 'vitest' + +import type { RosterRow } from './types' + +const routeMock = vi.fn<() => { connectionId: string; profile: string } | null>(() => null) + +vi.mock('@hermes/plugin-sdk', () => ({ + host: { requestProfile: vi.fn() }, + resolveSiblingWsUrl: vi.fn() +})) + +vi.mock('./routing', () => ({ + botConnectionRoute: () => routeMock() +})) + +import { isEventForBotScreen } from './screen-connection' + +const bot = { name: 'ops' } as RosterRow +const key = '/home/hermes/.hermes' + +describe('isEventForBotScreen', () => { + it('ignores a same-profile-path event that arrived from another host', () => { + routeMock.mockReturnValue({ connectionId: 'conn-a', profile: 'ops' }) + + const fromB = { connectionId: 'conn-b', payload: { profile_key: key }, type: 'display.lease' } + const fromA = { connectionId: 'conn-a', payload: { profile_key: key }, type: 'display.lease' } + + expect(isEventForBotScreen(bot, fromB, key)).toBe(false) + expect(isEventForBotScreen(bot, fromA, key)).toBe(true) + }) + + it('still matches the untagged local socket for a local bot', () => { + routeMock.mockReturnValue({ connectionId: 'local', profile: 'ops' }) + + expect(isEventForBotScreen(bot, { payload: { profile_key: key }, type: 'display.lease' }, key)).toBe(true) + expect(isEventForBotScreen(bot, { payload: { profile_key: '/other' }, type: 'display.lease' }, key)).toBe(false) + }) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-connection.ts b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts new file mode 100644 index 0000000000000..671fac03dd7e7 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-connection.ts @@ -0,0 +1,170 @@ +/** + * Bot Screen — connection plumbing for a bot's Bot Desktop (the headless Xfce + * screen its computer_use drives on the gateway host). + * + * Two legs share one authenticated origin: JSON-RPC (`display.*`) rides the + * bot's pooled gateway socket through `host.requestProfile`; the RFB stream + * rides a SIBLING WebSocket to `/api/display/ws`, minted per attach by + * `display.observe` (single-use, 30 s). noVNC's Websock takes ownership of the + * socket it is handed, so it can never share the JSON-RPC one — same reason + * voice playback opens `/api/audio/speak-stream` beside `/api/ws`. + */ + +import { host, resolveSiblingWsUrl } from '@hermes/plugin-sdk' +import type { PluginProfileRoute, RpcEvent } from '@hermes/plugin-sdk' + +import { botConnectionRoute } from './routing' +import type { RosterRow } from './types' + +export interface DisplayLease { + holder: 'agent' | 'human' + /** Raw holder id — only older backends still broadcast it; newer ones send `viewer_hash`. */ + viewer_id: null | string + /** First 12 hex of sha256(viewer_id): names the holder without leaking a usable id. */ + viewer_hash?: null | string + since: number + reason: string + pending_handoff: null | string + /** Monotonic per transition; a lower epoch is an older snapshot, never newer truth. */ + epoch?: number +} + +export interface DisplayStatus { + profile: string + profile_key: string + supported: boolean + installed: boolean + missing: string[] + running: boolean + pid: null | number + display: null | string + socket: null | string + geometry: string + install_command: null | string + lease: DisplayLease +} + +export interface DisplayThumbnail { + data_url: string | null + /** Set while a human holds the screen: the frame is withheld, not missing. */ + suppressed?: 'human_has_control' | null +} + +export interface DisplayObserveResult extends DisplayStatus { + ticket: string + path: string + /** Server-minted per attach: the only id the lease will ever be compared against. */ + viewer_id: string +} + +/** This window's identity for one attach: the minted id plus its lease-payload hash. */ +export interface ScreenViewer { + id: string + hash: string +} + +const VIEWER_HASH_HEX = 12 + +/** `viewer_hash` as the lease broadcasts it: first 12 hex of sha256(viewer_id). */ +export async function viewerHash(viewerId: string): Promise { + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(viewerId)) + + return Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, '0')) + .join('') + .slice(0, VIEWER_HASH_HEX) +} + +/** Does `viewer` (this window's attach) hold `lease`? Newer backends name the + * holder by hash only; a payload still carrying the raw id is compared raw. */ +export function leaseHeldBy(lease: DisplayLease | null | undefined, viewer: ScreenViewer | null | undefined): boolean { + if (!lease || !viewer || lease.holder !== 'human') { + return false + } + + return lease.viewer_id != null ? lease.viewer_id === viewer.id : lease.viewer_hash === viewer.hash +} + +/** JSON-RPC method-not-found: the bot's Hermes predates the `display.*` surface. */ +export function isDisplayUnavailable(error: unknown): boolean { + const record = typeof error === 'object' && error !== null ? (error as { code?: unknown; message?: unknown }) : null + + if (record?.code === -32601) { + return true + } + + const message = typeof record?.message === 'string' ? record.message.toLowerCase() : '' + + return message.includes('method not found') || message.includes('method-not-found') +} + +/** Bare-profile fallback so a v1 local bot (no registry route) still resolves. */ +export function botScreenRoute(bot: RosterRow): PluginProfileRoute | string { + return botConnectionRoute(bot) ?? bot.name +} + +/** + * Does a `display.*` event belong to `bot`'s screen? Two hosts can share the same + * `~/.hermes` path, so the profile key alone is ambiguous: the event must also have + * arrived on the bot's registry connection (local/legacy events carry no tag). + */ +export function isEventForBotScreen(bot: RosterRow, event: RpcEvent, profileKey: null | string | undefined): boolean { + const payload = event.payload as { profile_key?: string } | undefined + + if (!profileKey || payload?.profile_key !== profileKey) { + return false + } + + const expected = botConnectionRoute(bot)?.connectionId ?? null + const actual = event.connectionId ?? null + + return expected === actual || (expected === 'local' && actual === null) +} + +export function displayRequest(bot: RosterRow, method: string, params: Record = {}): Promise { + return host.requestProfile(botScreenRoute(bot), method, params) +} + +/** + * Hold the bot's pooled gateway socket open across a `display.*` sequence. The + * SDK disposes an inactive registry-routed socket once its request count hits + * zero, so without this the `display.install.*` / `display.lease` events that + * follow the request never arrive. Feature-detected: older hosts (and local + * routes, which never close) get a no-op release. + */ +export async function retainBotScreen(bot: RosterRow): Promise<() => void> { + const noop = () => undefined + + if (typeof host.retainProfile !== 'function') { + return noop + } + + try { + const release = await host.retainProfile(botScreenRoute(bot)) + + return typeof release === 'function' ? release : noop + } catch { + return noop + } +} + +/** + * Resolve the RFB WebSocket URL for `bot`: the bot's gateway `/api/ws` origin + * (fresh credential for OAuth remotes) with the path swapped for the display + * bridge and the single-use display ticket attached. + */ +export async function resolveScreenWsUrl(bot: RosterRow, ticket: string): Promise { + const route = botConnectionRoute(bot) + + // The /api/ws credential authenticated the RPC that minted the display ticket; + // the bridge authenticates on the ticket alone, so the gateway credential is + // dropped rather than spending a second one-shot ticket. + const url = new URL( + await resolveSiblingWsUrl({ connectionId: route?.connectionId ?? null, profile: route?.profile ?? bot.name }, '/api/display/ws', { + stripGatewayCredential: true + }) + ) + + url.searchParams.set('display_ticket', ticket) + + return url.toString() +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx new file mode 100644 index 0000000000000..647b866d4e553 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-hero.tsx @@ -0,0 +1,167 @@ +/** + * Screen hero — the big preview of a bot's computer at the top of its pane. + * + * Shows a live thumbnail (one `display.thumbnail` JPEG every few seconds while + * the screen runs and the hero is on screen), or an honest placeholder for + * stopped / not installed / unsupported. Clicking it opens the full Screen pane + * where the user can take over. + */ + +import { Codicon } from '@hermes/plugin-sdk' +import { useEffect, useRef, useState } from 'react' + +import { botSelectionKey } from './data' +import { useBots } from './i18n' +import { displayRequest, type DisplayThumbnail } from './screen-connection' +import { openBotScreen } from './screen-open' +import { type PortalTone, useScreenPortalState } from './screen-portal' +import type { BotMeta, RosterRow } from './types' + +const REFRESH_MS = 4000 +const STALE_AFTER = 3 + +function useLiveThumbnail(bot: RosterRow, running: boolean) { + const [dataUrl, setDataUrl] = useState(null) + // Consecutive failed refreshes; past STALE_AFTER the frame is shown dimmed as "last seen" so a + // dead gateway never keeps looking live. Success resets it. + const [misses, setMisses] = useState(0) + // The backend withholds frames while a human holds the screen; that is a + // deliberate answer, not a failed refresh, so it never ages into "stale". + const [suppressed, setSuppressed] = useState(false) + const boxRef = useRef(null) + + useEffect(() => { + if (!running) { + setDataUrl(null) + setMisses(0) + setSuppressed(false) + + return + } + + let cancelled = false + let timer: number | null = null + let visible = true + + const observer = + typeof IntersectionObserver === 'undefined' + ? null + : new IntersectionObserver(entries => { + visible = entries.some(entry => entry.isIntersecting) + }) + + if (observer && boxRef.current) { + observer.observe(boxRef.current) + } + + const tick = () => { + if (cancelled) { + return + } + + if (!visible || document.hidden) { + timer = window.setTimeout(tick, REFRESH_MS) + + return + } + + void displayRequest(bot, 'display.thumbnail') + .then(result => { + if (!cancelled) { + setDataUrl(result.data_url) + setSuppressed(result.suppressed === 'human_has_control') + setMisses(0) + } + }) + .catch(() => { + if (!cancelled) { + setMisses(prev => prev + 1) // the last good frame stays up, marked stale past STALE_AFTER + } + }) + .finally(() => { + if (!cancelled) { + timer = window.setTimeout(tick, REFRESH_MS) + } + }) + } + + tick() + + return () => { + cancelled = true + observer?.disconnect() + + if (timer !== null) { + window.clearTimeout(timer) + } + } + }, [bot, running]) + + return { dataUrl, boxRef, stale: misses >= STALE_AFTER, suppressed } +} + +const TONE_RING: Partial> = { + human: 'ring-2 ring-red-500/80', + other: 'ring-2 ring-amber-500/70' +} + +export function ScreenHero({ bot, meta }: { bot: RosterRow; meta?: BotMeta | null }) { + // A profile switch must discard the previous owner's pixels before painting. + return +} + +function ScreenHeroContent({ bot, meta }: { bot: RosterRow; meta?: BotMeta | null }) { + const t = useBots() + const { tone } = useScreenPortalState(bot) + const running = tone === 'live' || tone === 'human' || tone === 'other' + const { dataUrl, boxRef, stale, suppressed } = useLiveThumbnail(bot, running) + + if (tone === 'unsupported' || tone === 'unavailable') { + return null + } + + const caption = suppressed ? t.screen.heroSuppressed : stale ? t.screen.heroStale : { + live: t.screen.portalWatching, + human: t.screen.portalYouControl, + other: t.screen.portalOtherControls, + off: t.screen.heroStopped, + missing: t.screen.heroNotInstalled, + unsupported: t.screen.portalUnsupported, + unavailable: t.screen.portalUnavailable, + unknown: t.screen.heroConnecting + }[tone] + + const cta = running ? t.screen.heroOpenLive : tone === 'missing' ? t.screen.heroInstall : tone === 'off' ? t.screen.heroStart : '' + + return ( + + ) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx new file mode 100644 index 0000000000000..1f057dacce53e --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-install-retention.test.tsx @@ -0,0 +1,107 @@ +/** + * An INACTIVE registry-routed bot's pooled socket is disposed by the SDK as soon + * as its request count hits zero — so `display.install.log/done` (and the + * pane's `display.lease` events) would never arrive. The install card must hold + * a retention from before `display.install` until the done event lands. + */ + +import { act, fireEvent, render } from '@testing-library/react' +import { beforeEach, expect, it, vi } from 'vitest' + +import type { DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +const calls = vi.hoisted(() => [] as string[]) + +vi.mock('@hermes/plugin-sdk', async () => { + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Button: ({ children, ...props }: React.ButtonHTMLAttributes) => , + Codicon: () => null, + GlyphSpinner: () => null, + resolveSiblingWsUrl: vi.fn(), + host: { + onEvent: onGatewayEvent, + requestProfile: vi.fn(async (_route: unknown, method: string) => { + calls.push(`request:${method}`) + + return {} + }), + retainProfile: vi.fn(async () => { + calls.push('retain') + + return () => { + calls.push('release') + } + }) + } + } +}) +vi.mock('./routing', () => ({ + botConnectionRoute: () => ({ connectionId: 'host-a', profile: 'ops', targetProfile: 'ops' }) +})) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + notInstalledTitle: 'Missing', + notInstalledBody: 'Body', + installHint: 'Hint', + install: 'Install on host', + installing: 'Installing', + installCancelled: 'Cancelled', + installFailed: 'Failed', + noPackageManager: 'None' + } + }) +})) + +// eslint-disable-next-line no-restricted-imports +import { emitGatewayEvent } from '../../contrib/events' + +import { ScreenInstallCard } from './screen-install' + +const bot: RosterRow = { name: 'ops', sourceScoped: true, connectionId: 'host-a', connectionKind: 'remote' } + +const status: DisplayStatus = { + profile: 'ops', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: false, + missing: ['tigervnc'], + running: false, + pid: null, + display: null, + socket: null, + geometry: '1440x900', + install_command: 'sudo apt-get install -y tigervnc-standalone-server', + lease: { holder: 'agent', viewer_id: null, pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + calls.length = 0 +}) + +it('retains the bot socket before display.install and releases it when the done event lands', async () => { + const onInstalled = vi.fn() + const view = render() + + await act(async () => { + fireEvent.click(view.getByText('Install on host')) + }) + expect(calls).toEqual(['retain', 'request:display.install']) + + act(() => + emitGatewayEvent({ + type: 'display.install.done', + connectionId: 'host-a', + profile: 'ops', + payload: { profile_key: status.profile_key, code: 0, status: { ...status, installed: true } } + }) + ) + expect(calls).toEqual(['retain', 'request:display.install', 'release']) + expect(onInstalled).toHaveBeenCalledTimes(1) + view.unmount() + // Unmount after done must not double-release. + expect(calls.filter(call => call === 'release')).toHaveLength(1) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-install.tsx b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx new file mode 100644 index 0000000000000..f5509527c5fb7 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-install.tsx @@ -0,0 +1,125 @@ +/** + * Bot Screen install card — installs the TigerVNC + Xfce packages on the bot's + * gateway host from inside Hermes Desktop. + * + * `display.install` starts the distro package command on the host; sudo, when + * needed, arrives as the same masked password card the terminal tool uses + * (`display.install.sudo.request`), so the password never touches this pane. + * Output streams back as `display.install.log`; `display.install.done` carries + * a fresh status the caller uses to flip the pane to "Start screen". + */ + +import { Button, Codicon, GlyphSpinner, host } from '@hermes/plugin-sdk' +import type { RpcEvent } from '@hermes/plugin-sdk' +import { useCallback, useEffect, useRef, useState } from 'react' + +import { useBots } from './i18n' +import { displayRequest, type DisplayStatus, isEventForBotScreen, retainBotScreen } from './screen-connection' +import type { RosterRow } from './types' + +const LOG_KEEP = 200 + +interface ScreenInstallCardProps { + bot: RosterRow + status: DisplayStatus + onInstalled: (status: DisplayStatus) => void +} + +export function ScreenInstallCard({ bot, status, onInstalled }: ScreenInstallCardProps) { + const t = useBots() + const [phase, setPhase] = useState<'idle' | 'running' | 'failed'>('idle') + const [log, setLog] = useState([]) + const [error, setError] = useState(null) + const logEnd = useRef(null) + // Keeps the bot's socket open from display.install until done/failed: the log + // and done events ride that socket, and the SDK closes an idle one otherwise. + const retention = useRef<(() => void) | null>(null) + + const releaseRetention = useCallback(() => { + retention.current?.() + retention.current = null + }, []) + + useEffect(() => releaseRetention, [releaseRetention]) + + useEffect(() => { + logEnd.current?.scrollIntoView({ block: 'end' }) + }, [log]) + + useEffect(() => { + const offLog = host.onEvent('display.install.log', (event: RpcEvent) => { + const payload = event.payload as { line?: string } | undefined + + if (isEventForBotScreen(bot, event, status.profile_key) && typeof payload?.line === 'string') { + const line = payload.line + setLog(prev => (prev.length >= LOG_KEEP ? [...prev.slice(1), line] : [...prev, line])) + } + }) + + const offDone = host.onEvent('display.install.done', (event: RpcEvent) => { + const payload = event.payload as { code?: number; status?: DisplayStatus } | undefined + + if (!payload || !isEventForBotScreen(bot, event, status.profile_key)) { + return + } + + releaseRetention() + + if (payload.code === 0 && payload.status?.installed) { + setPhase('idle') + onInstalled(payload.status) + } else { + setPhase('failed') + setError(payload.code === -1 ? t.screen.installCancelled : t.screen.installFailed) + } + }) + + return () => { + offLog() + offDone() + } + }, [bot, onInstalled, releaseRetention, status.profile_key, t.screen.installCancelled, t.screen.installFailed]) + + const install = useCallback(async () => { + setPhase('running') + setLog([]) + setError(null) + + try { + retention.current = await retainBotScreen(bot) + await displayRequest(bot, 'display.install') + } catch (err) { + releaseRetention() + setPhase('failed') + setError(err instanceof Error ? err.message : String(err)) + } + }, [bot, releaseRetention]) + + return ( +
+
+
{t.screen.notInstalledTitle}
+
{t.screen.notInstalledBody}
+ {status.install_command ? ( + {status.install_command} + ) : ( +
{t.screen.noPackageManager}
+ )} + {status.install_command ? ( + + ) : null} + {log.length > 0 ? ( +
+            {log.join('\n')}
+            
+
+ ) : null} + {error ?
{error}
: null} +
{t.screen.installHint}
+
+
+ ) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx new file mode 100644 index 0000000000000..00b65c11e8f2f --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-isolation.test.tsx @@ -0,0 +1,214 @@ +import { act, fireEvent, render, renderHook } from '@testing-library/react' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import type { RosterRow } from './types' + +vi.mock('@hermes/plugin-sdk', async () => { + const { useStore } = await import('@nanostores/react') + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Codicon: () => null, + useValue: useStore, + resolveSiblingWsUrl: vi.fn(), + host: { onEvent: vi.fn(onGatewayEvent), requestProfile: vi.fn() } + } +}) +vi.mock('./data', async () => { + const { atom } = await import('nanostores') + + return { + $lastRoster: atom([]), + botSelectionKey: (bot: RosterRow) => + bot.sourceScoped || bot.remoteSource ? `${bot.connectionId}::${bot.name}` : bot.name + } +}) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + portalTitle: 'Screen', + portalWatching: 'Live', + portalYouControl: 'You control', + portalOtherControls: 'Other viewer', + heroOpenLive: 'Open live', + heroStale: 'Last seen', + heroSuppressed: 'Hidden while someone has control', + portalUnavailable: 'Update the bot', + heroConnecting: 'Connecting' + } + }) +})) +vi.mock('./screen-open', () => ({ openBotScreen: vi.fn() })) + +import { host } from '@hermes/plugin-sdk' + +// Exercise the real event bus in this integration test, not a copied dispatcher. +// eslint-disable-next-line no-restricted-imports +import { emitGatewayEvent } from '../../contrib/events' + +import { $lastRoster } from './data' +import type { DisplayStatus } from './screen-connection' +import { ScreenHero } from './screen-hero' +import { openBotScreen } from './screen-open' +import { ProfileGroupScreenPortal, useScreenPortalState } from './screen-portal' +import { $screenState, setScreenStatus } from './screen-state' + +const botA: RosterRow = { name: 'default', sourceScoped: true, connectionId: 'host-a', connectionKind: 'remote' } +const botB: RosterRow = { ...botA, connectionId: 'host-b' } + +const status: DisplayStatus = { + profile: 'default', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 42, + display: ':20', + socket: '/tmp/rfb.sock', + geometry: '1440x900', + install_command: null, + lease: { holder: 'agent', viewer_id: null, pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + $screenState.set({}) + $lastRoster.set([]) + vi.mocked(host.requestProfile).mockReset() + vi.mocked(openBotScreen).mockClear() + vi.spyOn(globalThis.document, 'hidden', 'get').mockReturnValue(false) +}) + +afterEach(() => { + vi.restoreAllMocks() + vi.useRealTimers() +}) + +it('applies lease events only from the owning host even when profile paths match', () => { + setScreenStatus(botA, status) + const view = renderHook(() => useScreenPortalState(botA)) + const human = { ...status.lease, holder: 'human' as const, viewer_id: 'this-viewer' } + + const emit = (connectionId: string, profileKey = status.profile_key) => + act(() => + emitGatewayEvent({ + type: 'display.lease', + connectionId, + profile: 'default', + payload: { profile_key: profileKey, lease: human } + }) + ) + + emit('host-b') + expect(view.result.current.lease?.holder).toBe('agent') + emit('host-a', '/another/profile') + expect(view.result.current.lease?.holder).toBe('agent') + emit('host-a') + expect(view.result.current.lease).toEqual(human) + view.unmount() +}) + +it('does not match a legacy profile group to a same-named remote bot', () => { + const legacy: RosterRow = { name: 'default' } + $lastRoster.set([botB, legacy]) + setScreenStatus(botB, status) + setScreenStatus(legacy, status) + const view = render() + + fireEvent.click(view.getByRole('button')) + expect(openBotScreen).toHaveBeenCalledWith(legacy, null) + view.rerender() + fireEvent.click(view.getByRole('button')) + expect(openBotScreen).toHaveBeenLastCalledWith(botB, null) + view.unmount() +}) + +it.each(['pending', 'rejected'])('never displays host A pixels under host B while B is %s', async state => { + setScreenStatus(botA, status) + setScreenStatus(botB, status) + vi.mocked(host.requestProfile) + .mockResolvedValueOnce({ data_url: 'data:image/jpeg;base64,HOST_A' }) + .mockImplementationOnce(() => (state === 'pending' ? new Promise(() => {}) : Promise.reject(new Error('offline')))) + const view = render() + await act(async () => {}) + expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_A') + + view.rerender() + await act(async () => {}) + expect(view.container.querySelector('img')).toBeNull() + view.unmount() +}) + +it('discards a late thumbnail from the previous owner and retains a same-owner frame on refresh failure', async () => { + vi.useFakeTimers() + setScreenStatus(botA, status) + setScreenStatus(botB, status) + let finishA!: (value: unknown) => void + vi.mocked(host.requestProfile) + .mockImplementationOnce( + () => + new Promise(resolve => { + finishA = resolve + }) + ) + .mockResolvedValueOnce({ data_url: 'data:image/jpeg;base64,HOST_B' }) + .mockRejectedValue(new Error('offline')) + const view = render() + view.rerender() + await act(async () => {}) + await act(async () => { + finishA({ data_url: 'data:image/jpeg;base64,HOST_A' }) + }) + expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_B') + await act(async () => { + await vi.advanceTimersByTimeAsync(12_000) + }) + expect(view.container.querySelector('img')?.getAttribute('src')).toContain('HOST_B') + expect(view.getByRole('button').getAttribute('aria-label')).toContain('Last seen') + view.unmount() +}) + +it('captions a suppressed thumbnail as hidden-while-controlled and never ages it into stale', async () => { + vi.useFakeTimers() + setScreenStatus(botA, status) + vi.mocked(host.requestProfile).mockResolvedValue({ data_url: null, suppressed: 'human_has_control' }) + const view = render() + await act(async () => {}) + expect(view.getByRole('button').getAttribute('aria-label')).toContain('Hidden while someone has control') + + await act(async () => { + await vi.advanceTimersByTimeAsync(20_000) + }) + expect(view.getByRole('button').getAttribute('aria-label')).toContain('Hidden while someone has control') + expect(view.getByRole('button').getAttribute('aria-label')).not.toContain('Last seen') + view.unmount() +}) + +it('settles on an older backend without display.*: portal tone is unavailable and the hero renders nothing', async () => { + vi.mocked(host.requestProfile).mockRejectedValue(Object.assign(new Error('Method not found: display.status'), { code: -32601 })) + const hook = renderHook(() => useScreenPortalState(botA)) + await act(async () => {}) + expect(hook.result.current.tone).toBe('unavailable') + hook.rerender() + await act(async () => {}) + expect(vi.mocked(host.requestProfile)).toHaveBeenCalledTimes(1) + hook.unmount() + + const view = render() + await act(async () => {}) + expect(view.container.firstChild).toBeNull() + view.unmount() +}) + +it('a sidebar group portal keeps its lease subscription across parent re-renders (no per-paint row rebuild)', async () => { + vi.mocked(host.requestProfile).mockResolvedValue(status) + const view = render() + await act(async () => {}) + const subscriptions = vi.mocked(host.onEvent).mock.calls.length + + view.rerender() + view.rerender() + await act(async () => {}) + expect(vi.mocked(host.onEvent).mock.calls.length).toBe(subscriptions) + view.unmount() +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-open.tsx b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx new file mode 100644 index 0000000000000..9d05539c4d889 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-open.tsx @@ -0,0 +1,46 @@ +/** + * Open a bot's Screen as a main-window workspace tab (host.openWorkspace), + * one tab per bot; a second open refocuses the existing tab. + */ + +import { host } from '@hermes/plugin-sdk' + +import { botSelectionKey } from './data' +import { botsText } from './i18n' +import { displayName } from './labels' +import { BotScreenPane } from './screen-pane' +import { ID } from './shared' +import type { BotMeta, RosterRow } from './types' + +const openTabs = new Map void>() + +export function screenPaneId(bot: RosterRow): string { + return `plugin-workspace:${ID}:screen:${botSelectionKey(bot)}` +} + +export function openBotScreen(bot: RosterRow, meta?: BotMeta | null): void { + if (typeof host.openWorkspace !== 'function') { + host.notify({ kind: 'info', message: botsText().screen.openNeedsUpdate }) + + return + } + + const key = botSelectionKey(bot) + + if (openTabs.has(key)) { + host.revealPane(screenPaneId(bot)) + + return + } + + const close = host.openWorkspace(`${ID}:screen:${key}`, { + title: `${displayName(bot, meta ?? null)} · ${botsText().screen.title}`, + minWidth: '28rem', + render: () => , + onClose: () => { + openTabs.delete(key) + } + }) + + openTabs.set(key, close) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx new file mode 100644 index 0000000000000..95eaa59522981 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane-lifecycle.test.tsx @@ -0,0 +1,201 @@ +import { act, fireEvent, render, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import type { DisplayStatus } from './screen-connection' +import type * as ScreenConnection from './screen-connection' +import type { RosterRow } from './types' + +const sockets = vi.hoisted( + () => [] as Array<{ closeCodes: number[]; closed: boolean; close: (code?: number) => void; serverClose: (code: number) => void }> +) + +const rfbs = vi.hoisted(() => [] as Array<{ emit: (type: string, detail?: unknown) => void }>) +const retention = vi.hoisted(() => ({ held: 0 })) + +vi.mock('@hermes/plugin-sdk', async () => { + const { useStore } = await import('@nanostores/react') + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Button: ({ children, ...props }: React.ButtonHTMLAttributes) => ( + + ), + Codicon: () => null, + GlyphSpinner: () => null, + EmptyState: () => null, + useValue: useStore, + host: { + onEvent: onGatewayEvent, + retainProfile: async () => { + retention.held += 1 + + return () => { + retention.held -= 1 + } + } + } + } +}) +vi.mock('./routing', () => ({ + botConnectionRoute: () => ({ connectionId: 'host-a', profile: 'default', targetProfile: 'default' }) +})) +vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name })) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + title: 'Screen', + controlTaken: 'Another viewer took control', + youControl: 'You control', + handBack: 'Hand back', + takeOver: 'Take over', + reconnect: 'Reconnect', + streamLost: 'Stream lost' + } + }) +})) +vi.mock('./screen-connection', async importActual => ({ + // Real pure helpers (viewerHash / leaseHeldBy); only the gateway legs are faked. + ...(await importActual()), + displayRequest: vi.fn(), + resolveScreenWsUrl: vi.fn(async () => 'ws://localhost/api/display/ws'), + isEventForBotScreen: () => false +})) +vi.mock('@novnc/novnc', () => ({ + default: class { + private listeners = new Map void>>() + constructor( + _target: HTMLElement, + private socket: { close: () => void } + ) { + rfbs.push(this) + } + emit(type: string, detail?: unknown) { + for (const listener of this.listeners.get(type) ?? []) { + listener({ detail }) + } + } + addEventListener(type: string, callback: (event: { detail?: unknown }) => void) { + this.listeners.set(type, [...(this.listeners.get(type) ?? []), callback]) + + if (type === 'connect') { + queueMicrotask(() => callback({})) + } + } + // noVNC 1.7 disconnects its WebSocket without a close code. + disconnect() { + this.socket.close() + } + focus() {} + } +})) + +import { displayRequest } from './screen-connection' +import { BotScreenPane } from './screen-pane' +import { $screenState } from './screen-state' + +const bot: RosterRow = { name: 'default' } + +const status: DisplayStatus = { + profile: 'default', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 42, + display: ':20', + socket: '/tmp/rfb.sock', + geometry: '1440x900', + install_command: null, + lease: { holder: 'human', viewer_id: 'this-viewer', pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + $screenState.set({}) + sockets.length = 0 + rfbs.length = 0 + retention.held = 0 + vi.mocked(displayRequest) + .mockReset() + .mockResolvedValue({ ...status, ticket: 'test-ticket', viewer_id: 'this-viewer' }) + vi.stubGlobal( + 'WebSocket', + class { + closeCodes: number[] = [] + closed = false + private onClose: Array<(event: { code: number }) => void> = [] + constructor() { + sockets.push(this) + } + addEventListener(type: string, listener: (event: { code: number }) => void) { + if (type === 'close') { + this.onClose.push(listener) + } + } + // The bridge closing us: the raw close frame reaches our listener, then noVNC + // reports a statusless `disconnect` — the code is only on the socket event. + serverClose(code: number) { + this.closed = true + + for (const listener of this.onClose) { + listener({ code }) + } + } + close(code?: number) { + // Subsequent close calls cannot replace the frame already sent to the server. + if (this.closed) { + return + } + + this.closed = true + this.closeCodes.push(code ?? 1005) + } + } + ) +}) + +afterEach(() => vi.unstubAllGlobals()) + +it('sends an intentional close before noVNC can send its statusless close on pane unmount', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + view.unmount() + expect(sockets[0].closeCodes).toEqual([1000]) +}) + +it('pins the bot socket for the attach lifetime and lets go on unmount', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + expect(retention.held).toBe(1) + fireEvent.click(view.getByTitle('Reconnect')) + await waitFor(() => expect(sockets).toHaveLength(2)) + expect(retention.held).toBe(1) + view.unmount() + expect(retention.held).toBe(0) +}) + +it('does not hand back while replacing a stream to reconnect the same viewer', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + fireEvent.click(view.getByTitle('Reconnect')) + await waitFor(() => expect(sockets).toHaveLength(2)) + expect(sockets[0].closeCodes).toEqual([1005]) + expect(sockets[1].closed).toBe(false) + view.unmount() +}) + +it('shows the control-taken overlay from the bridge close code, which noVNC does not forward', async () => { + const view = render() + await waitFor(() => expect(sockets).toHaveLength(1)) + await act(async () => {}) + + act(() => { + sockets[0].serverClose(4000) + rfbs[0].emit('disconnect', { clean: true }) + }) + expect(view.getByText('Another viewer took control')).toBeTruthy() + view.unmount() +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx new file mode 100644 index 0000000000000..1dea30e2a4e58 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-pane.tsx @@ -0,0 +1,364 @@ +/** + * Bot Screen pane — live view of a bot's headless desktop with Take over / Hand back. + * + * State authority: the BACKEND owns runtime + lease (`display.status`, pushed + * as `display.lease` events); this pane paints a cache of it. The RFB stream + * is a sibling WebSocket handed to noVNC's RFB; `viewOnly` here is UX only — + * the gateway drops input from anyone but the lease holder. + * + * Every attach spends a single-use ticket, so a lease flip that the bridge + * answers with close 4000 (`control-taken`) simply re-attaches in watch mode. + */ + +import { Button, Codicon, EmptyState, GlyphSpinner, host, useValue } from '@hermes/plugin-sdk' +import type { RpcEvent } from '@hermes/plugin-sdk' +import { useCallback, useEffect, useRef, useState } from 'react' + +import { useBots } from './i18n' +import { type DisplayLease, type DisplayObserveResult, displayRequest, type DisplayStatus, isDisplayUnavailable, isEventForBotScreen, leaseHeldBy, resolveScreenWsUrl, retainBotScreen, viewerHash } from './screen-connection' +import { ScreenInstallCard } from './screen-install' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenUnavailable, setScreenViewer } from './screen-state' +import type { RosterRow } from './types' + +type RfbLike = { + viewOnly: boolean + scaleViewport: boolean + resizeSession: boolean + focusOnClick: boolean + background: string + qualityLevel: number + addEventListener: (type: string, handler: (event: { detail?: { clean?: boolean; reason?: string } }) => void) => void + disconnect: () => void + focus: () => void +} + +type ConnState = 'idle' | 'attaching' | 'live' | 'control-taken' | 'error' + +/** Bridge close code when another viewer took the lease (mirrors tui_gateway display bridge). */ +const CLOSE_CONTROL_TAKEN = 4000 + +async function loadRfb(): Promise) => RfbLike> { + const mod = (await import('@novnc/novnc')) as unknown as { default: new (...args: never[]) => RfbLike } + + return mod.default as unknown as new (target: HTMLElement, socket: WebSocket, options?: Record) => RfbLike +} + +export function BotScreenPane({ bot }: { bot: RosterRow }) { + const t = useBots() + const screen = useValue($screenState) + const state = screenStateFor(screen, bot) + const status = state?.status ?? null + const lease = state?.lease ?? status?.lease ?? null + // The server mints this window's viewer id per attach (`display.observe`); the lease + // names its holder by hash, so a reload can never inherit a stale holder's authority. + const viewer = state?.viewer ?? null + const iHold = leaseHeldBy(lease, viewer) + + const canvasHost = useRef(null) + const rfb = useRef(null) + const socket = useRef(null) + // Pins the bot's pooled gateway socket for the attach lifetime so display.lease + // events keep arriving for an inactive registry-routed bot. + const retention = useRef<(() => void) | null>(null) + const [conn, setConn] = useState('idle') + const [error, setError] = useState(null) + const [busy, setBusy] = useState(false) + const attachGeneration = useRef(0) + + const refresh = useCallback(async () => { + try { + const next = await displayRequest(bot, 'display.status') + setScreenStatus(bot, next) + setError(null) + } catch (err) { + if (isDisplayUnavailable(err)) { + setScreenUnavailable(bot) + } + + setError(err instanceof Error ? err.message : String(err)) + } + }, [bot]) + + useEffect(() => { + void refresh() + + return host.onEvent('display.lease', (event: RpcEvent) => { + const payload = event.payload as { lease?: DisplayLease } | undefined + + if (payload?.lease && isEventForBotScreen(bot, event, status?.profile_key)) { + setScreenLease(bot, payload.lease) + } + }) + }, [bot, refresh, status?.profile_key]) + + const detach = useCallback((handBack = false) => { + attachGeneration.current += 1 + + // noVNC closes without a status. Intentional pane closure must send 1000 + // first; reconnect teardown must keep the human lease instead. + if (handBack) { + socket.current?.close(1000) + } + + rfb.current?.disconnect() + rfb.current = null + socket.current?.close() + socket.current = null + retention.current?.() + retention.current = null + }, []) + + const attach = useCallback(async () => { + if (!canvasHost.current) { + return + } + + detach() + const generation = attachGeneration.current + setConn('attaching') + setError(null) + + try { + // Load the client BEFORE dialing: noVNC's Websock installs its own `onopen`, so a socket that + // opened while the dynamic import was still in flight never hands it the open event. + const Rfb = await loadRfb() + const retain = await retainBotScreen(bot) + + if (generation !== attachGeneration.current) { + retain() + + return + } + + retention.current = retain + const observe = await displayRequest(bot, 'display.observe') + const minted = { id: observe.viewer_id, hash: await viewerHash(observe.viewer_id) } + setScreenStatus(bot, observe) + const url = await resolveScreenWsUrl(bot, observe.ticket) + + if (generation !== attachGeneration.current || !canvasHost.current) { + return + } + + setScreenViewer(bot, minted) + const ws = new WebSocket(url) + ws.binaryType = 'arraybuffer' + socket.current = ws + // noVNC 1.7's `disconnect` detail carries only {clean}; the bridge's verdict lives in + // the raw close frame (4000 = control-taken). Listen here, before RFB installs its + // own `onclose`, so the code is known by the time the disconnect event fires. + let closeCode = 0 + ws.addEventListener('close', event => { + closeCode = event.code + }) + const client = new Rfb(canvasHost.current, ws, { shared: true }) + client.scaleViewport = true + client.resizeSession = false + client.focusOnClick = true + client.background = 'transparent' + client.qualityLevel = 7 + client.viewOnly = !leaseHeldBy(observe.lease, minted) + client.addEventListener('connect', () => { + if (generation === attachGeneration.current) { + setConn('live') + } + }) + client.addEventListener('disconnect', event => { + // noVNC logs "Tried changing state of a disconnected RFB object" if we later call + // disconnect() on a client that already closed itself (eviction, stream loss). + if (rfb.current === client) { + rfb.current = null + } + + if (generation !== attachGeneration.current) { + return + } + + const reason = event.detail?.reason ?? '' + + if (closeCode === CLOSE_CONTROL_TAKEN || reason.includes('control-taken')) { + setConn('control-taken') + } else if (event.detail?.clean) { + setConn('idle') + } else { + setConn('error') + setError(reason || t.screen.streamLost) + } + + void refresh() + }) + rfb.current = client + } catch (err) { + if (generation === attachGeneration.current) { + setConn('error') + setError(err instanceof Error ? err.message : String(err)) + } + } + }, [bot, detach, refresh, t.screen.streamLost]) + + // Visibility is not lifecycle: the stream stays attached while the pane is + // hidden; only unmount tears it down (and hands control back server-side). + useEffect(() => () => detach(true), [detach]) + + useEffect(() => { + if (status?.running && conn === 'idle') { + void attach() + } + }, [attach, conn, status?.running]) + + useEffect(() => { + if (rfb.current) { + rfb.current.viewOnly = !iHold + + if (iHold) { + rfb.current.focus() + } + } + }, [iHold]) + + const start = useCallback(async () => { + setBusy(true) + + try { + const next = await displayRequest(bot, 'display.start') + setScreenStatus(bot, next) + setConn('idle') + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [bot]) + + const takeOver = useCallback(async () => { + setBusy(true) + + try { + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.acquire', { viewer_id: viewer?.id }) + setScreenLease(bot, result.lease) + + if (conn !== 'live') { + void attach() + } + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, [attach, bot, conn, viewer?.id]) + + // `force` is the escape hatch for a lease this window no longer owns (a reload + // minted a fresh viewer id; the old one still holds): the server refuses a + // plain release from anyone but the holder. + const handBack = useCallback( + async (force = false) => { + setBusy(true) + + try { + const params = force ? { force: true } : { viewer_id: viewer?.id } + const result = await displayRequest<{ lease: DisplayLease }>(bot, 'display.lease.release', params) + setScreenLease(bot, result.lease) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setBusy(false) + } + }, + [bot, viewer?.id] + ) + + if (state?.unavailable) { + return + } + + if (status && !status.supported) { + return + } + + if (status && !status.installed) { + return setScreenStatus(bot, next)} status={status} /> + } + + if (status && !status.running) { + return ( +
+
+
{t.screen.stoppedTitle}
+
{t.screen.stoppedBody}
+ + {error ?
{error}
: null} +
+
+ ) + } + + const humanOther = lease?.holder === 'human' && !iHold + + return ( +
+
+ + {t.screen.title} + {status?.display ? {status.display} · {status.geometry} : null} + + {lease?.pending_handoff ? ( + + {t.screen.handoffRequested} + + ) : lease?.holder === 'human' && lease.reason ? ( + // The agent's ask stays readable WHILE the human acts, not only before Take over. + + {lease.reason} + + ) : null} + {iHold ? ( + {t.screen.youControl} + ) : humanOther ? ( + {t.screen.otherControls} + ) : ( + {t.screen.agentControls} + )} + {iHold ? ( + + ) : ( + <> + {humanOther ? ( + + ) : null} + + + )} + +
+
+ {/* data-terminal: the same keyboard-ownership marker the terminal pane uses, so the app's + type-to-focus / bare-key shortcuts never steal keystrokes meant for the remote screen. + data-remote-screen: tells the ⌘W close-tab router this is NOT a local terminal tab — + the chord belongs to the remote desktop, nothing local should close. */} +
+ {conn === 'attaching' ? ( +
+ {t.screen.attaching} +
+ ) : null} + {conn === 'control-taken' ? ( +
{t.screen.controlTaken}
+ ) : null} + {conn === 'error' && error ? ( +
{error}
+ ) : null} +
+
+ ) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx new file mode 100644 index 0000000000000..7afa011f24364 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-portal.tsx @@ -0,0 +1,191 @@ +/** + * Screen portal — the compact "this bot's computer" box that sits above a + * bot's routines and on a gateway/profile group in the Sessions sidebar. + * One click opens the live Screen pane; the subtitle says whether the screen + * is live and who holds it, so the user knows before opening whether they + * are about to watch, take over, install or start. + * + * Reads the same per-bot cache the pane paints (`$screenState`) and refreshes + * it once on mount so a bot the user never opened still shows real state. + */ + +import { Codicon, host, useValue } from '@hermes/plugin-sdk' +import type { RpcEvent } from '@hermes/plugin-sdk' +import type { ProfileGroupRoute } from '@hermes/plugin-sdk' +import { useEffect, useMemo } from 'react' + +import { $lastRoster } from './data' +import { useBots } from './i18n' +import { resolveBotConnectionRoute } from './routing' +import { type DisplayLease, displayRequest, type DisplayStatus, isDisplayUnavailable, isEventForBotScreen, leaseHeldBy, type ScreenViewer } from './screen-connection' +import { openBotScreen } from './screen-open' +import { $screenState, screenStateFor, setScreenLease, setScreenStatus, setScreenUnavailable } from './screen-state' +import type { BotMeta, RosterRow } from './types' + +export type PortalTone = 'live' | 'human' | 'other' | 'off' | 'missing' | 'unsupported' | 'unavailable' | 'unknown' + +/** Pure: map cached status + lease (+ this window's minted viewer, if attached) to what the portal says. */ +export function portalTone(status: DisplayStatus | null, lease: DisplayLease | null, viewer: ScreenViewer | null = null, unavailable = false): PortalTone { + if (unavailable) { + return 'unavailable' + } + + if (!status) { + return 'unknown' + } + + if (!status.supported) { + return 'unsupported' + } + + if (!status.installed) { + return 'missing' + } + + if (!status.running) { + return 'off' + } + + if (lease?.holder === 'human') { + return leaseHeldBy(lease, viewer) ? 'human' : 'other' + } + + return 'live' +} + +const TONE_ICON: Record = { + live: 'device-desktop', + human: 'record-keys', + other: 'eye', + off: 'debug-stop', + missing: 'cloud-download', + unsupported: 'circle-slash', + unavailable: 'circle-slash', + unknown: 'device-desktop' +} + +const TONE_DOT: Record = { + live: 'bg-emerald-500', + human: 'bg-red-500', + other: 'bg-amber-500', + off: 'bg-(--ui-text-quaternary)', + missing: 'bg-(--ui-text-quaternary)', + unsupported: 'bg-(--ui-text-quaternary)', + unavailable: 'bg-(--ui-text-quaternary)', + unknown: 'bg-(--ui-text-quaternary)' +} + +export function useScreenPortalState(bot: RosterRow) { + const all = useValue($screenState) + const state = screenStateFor(all, bot) + const status = state?.status ?? null + const profileKey = status?.profile_key + + useEffect(() => { + if (status || state?.unavailable) { + return + } + + let cancelled = false + + void displayRequest(bot, 'display.status') + .then(next => { + if (!cancelled) { + setScreenStatus(bot, next) + } + }) + .catch((error: unknown) => { + // An older Hermes without display.* is a settled answer (hide the surface); + // an offline bot is transient and stays in its unknown state. + if (!cancelled && isDisplayUnavailable(error)) { + setScreenUnavailable(bot) + } + }) + + return () => { + cancelled = true + } + }, [bot, state?.unavailable, status]) + + useEffect( + () => + host.onEvent('display.lease', (event: RpcEvent) => { + const payload = event.payload as { profile_key?: string; lease?: DisplayLease } | undefined + + if (payload?.lease && isEventForBotScreen(bot, event, profileKey)) { + setScreenLease(bot, payload.lease) + } + }), + [bot, profileKey] + ) + + return { status, lease: state?.lease ?? null, tone: portalTone(status, state?.lease ?? null, state?.viewer ?? null, state?.unavailable) } +} + +export function ScreenPortal({ bot, meta, compact = false }: { bot: RosterRow; meta?: BotMeta | null; compact?: boolean }) { + const t = useBots() + const { status, tone } = useScreenPortalState(bot) + + const subtitle = { + live: t.screen.portalWatching, + human: t.screen.portalYouControl, + other: t.screen.portalOtherControls, + off: t.screen.portalStopped, + missing: t.screen.portalNotInstalled, + unsupported: t.screen.portalUnsupported, + unavailable: t.screen.portalUnavailable, + unknown: status?.display ?? '' + }[tone] + + if ((tone === 'unsupported' || tone === 'unavailable') && compact) { + return null + } + + return ( + + ) +} + +/** Sessions-sidebar variant: the gateway/profile group hands us its route; find the + * bot that owns it, or synthesize a scoped row so a profile outside the current + * roster filter still gets its portal (the portal only needs a routable row). */ +export function ProfileGroupScreenPortal({ route }: { route: ProfileGroupRoute }) { + const roster = useValue($lastRoster) + const { connectionId, profile } = route + + // Stable row identity: the portal's effects key on `bot`, so a synthesized row + // rebuilt every render would re-subscribe the lease listener on every sidebar paint. + const bot = useMemo( + () => + roster.find(row => { + const resolved = resolveBotConnectionRoute(row) + + return resolved.route + ? resolved.route.profile === profile && resolved.route.connectionId === (connectionId ?? 'local') + : row.name === profile && connectionId === null + }) ?? + (connectionId + ? ({ name: profile, sourceScoped: true, connectionId, connectionKind: connectionId === 'local' ? 'local' : 'remote' } as RosterRow) + : ({ name: profile } as RosterRow)), + [connectionId, profile, roster] + ) + + return +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts new file mode 100644 index 0000000000000..6e294921852ff --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.test.ts @@ -0,0 +1,61 @@ +/** + * Lease ordering: `display.lease` events and `display.status` replies race on + * the wire. A slower status reply describing an OLDER lease must never roll + * back the newer event — the backend's monotonic `epoch` is the tiebreak. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { DisplayLease, DisplayStatus } from './screen-connection' +import type { RosterRow } from './types' + +vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name })) + +import { $screenState, screenStateFor, setScreenLease, setScreenStatus } from './screen-state' + +const bot: RosterRow = { name: 'ops' } + +const agent: DisplayLease = { holder: 'agent', viewer_id: null, viewer_hash: null, since: 1, reason: '', pending_handoff: null, epoch: 3 } +const human: DisplayLease = { ...agent, holder: 'human', viewer_hash: 'abc123abc123', epoch: 4 } + +const statusWith = (lease: DisplayLease): DisplayStatus => ({ + profile: 'ops', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 1, + display: ':20', + socket: null, + geometry: '1440x900', + install_command: null, + lease +}) + +beforeEach(() => $screenState.set({})) + +describe('lease epoch ordering', () => { + it('a status reply carrying an older epoch does not roll back a newer lease event', () => { + setScreenLease(bot, human) + setScreenStatus(bot, statusWith(agent)) + + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(human) + // The status itself still lands — only its stale lease is ignored. + expect(screenStateFor($screenState.get(), bot)?.status?.running).toBe(true) + }) + + it('a lease event with an older epoch is ignored; a newer or epoch-less one applies', () => { + setScreenLease(bot, human) + setScreenLease(bot, agent) + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(human) + + const released = { ...agent, epoch: 5 } + setScreenLease(bot, released) + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(released) + + const legacy = { ...human, epoch: undefined } + setScreenLease(bot, legacy) + expect(screenStateFor($screenState.get(), bot)?.lease).toEqual(legacy) + }) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/screen-state.ts b/apps/desktop/src/plugins/hermes-bots/screen-state.ts new file mode 100644 index 0000000000000..06bfb8e6f8ddb --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-state.ts @@ -0,0 +1,89 @@ +/** + * Per-bot Bot Screen cache: backend truth (`display.status`) plus the last + * `display.lease` event, keyed by the bot's roster identity. Renderer-owned + * cache of backend state — never the authority. + */ + +import { atom } from 'nanostores' + +import { botSelectionKey } from './data' +import type { DisplayLease, DisplayStatus, ScreenViewer } from './screen-connection' +import type { RosterRow } from './types' + +export interface BotScreenState { + status: DisplayStatus | null + lease: DisplayLease | null + /** This window's server-minted identity for the bot's current attach; null until the pane observes. */ + viewer: ScreenViewer | null + /** The bot's Hermes has no `display.*` methods (older backend): nothing to check, ever. */ + unavailable?: boolean +} + +export const $screenState = atom>({}) + +export function screenStateFor(all: Record, bot: RosterRow): BotScreenState | null { + return all[botSelectionKey(bot)] ?? null +} + +/** A lease whose epoch is below the one we hold is a slower response about the + * past (a `display.status` reply overtaken by a `display.lease` event). Payloads + * without an epoch — older backends — are always applied. */ +function isOlderLease(prev: DisplayLease | null | undefined, next: DisplayLease): boolean { + return typeof next.epoch === 'number' && typeof prev?.epoch === 'number' && next.epoch < prev.epoch +} + +export function setScreenStatus(bot: RosterRow, status: DisplayStatus): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + const lease = status.lease && !isOlderLease(prev?.lease, status.lease) ? status.lease : (prev?.lease ?? null) + $screenState.set({ ...current, [key]: { status, lease, viewer: prev?.viewer ?? null } }) +} + +/** `display.status` answered method-not-found: remember it so no surface keeps "checking". */ +export function setScreenUnavailable(bot: RosterRow): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + + if (prev?.unavailable) { + return + } + + $screenState.set({ ...current, [key]: { status: null, lease: null, viewer: null, unavailable: true } }) +} + +export function setScreenLease(bot: RosterRow, lease: DisplayLease): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + + if (isOlderLease(prev?.lease, lease)) { + return + } + + if ( + prev?.lease && + prev.lease.holder === lease.holder && + prev.lease.viewer_id === lease.viewer_id && + prev.lease.viewer_hash === lease.viewer_hash && + prev.lease.pending_handoff === lease.pending_handoff + ) { + return + } + + $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease, viewer: prev?.viewer ?? null } }) +} + +/** Record the identity `display.observe` minted for this window's attach to `bot`. */ +export function setScreenViewer(bot: RosterRow, viewer: ScreenViewer | null): void { + const key = botSelectionKey(bot) + const current = $screenState.get() + const prev = current[key] + + if ((prev?.viewer ?? null) === viewer) { + return + } + + $screenState.set({ ...current, [key]: { status: prev?.status ?? null, lease: prev?.lease ?? null, viewer } }) +} diff --git a/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx b/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx new file mode 100644 index 0000000000000..04a782af7eb9d --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/screen-viewer-identity.test.tsx @@ -0,0 +1,153 @@ +/** + * Viewer identity is SERVER-MINTED: `display.observe` returns the id this attach + * is known by, and lease payloads name the holder by `viewer_hash` + * (sha256(viewer_id)[:12]) rather than the raw id. "I hold" must be derived + * from the minted id, never from a client-generated constant — otherwise a + * Desktop reload could claim (or lose) control it does not have. + */ + +import { act, render, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' + +import type { DisplayStatus } from './screen-connection' +import type * as ScreenConnection from './screen-connection' +import type { RosterRow } from './types' + +vi.mock('@hermes/plugin-sdk', async () => { + const { useStore } = await import('@nanostores/react') + const { onGatewayEvent } = await import('../../contrib/events') + + return { + Button: ({ children, ...props }: React.ButtonHTMLAttributes) => , + Codicon: () => null, + GlyphSpinner: () => null, + EmptyState: () => null, + useValue: useStore, + host: { onEvent: onGatewayEvent } + } +}) +vi.mock('./data', () => ({ botSelectionKey: (bot: RosterRow) => bot.name })) +vi.mock('./i18n', () => ({ + useBots: () => ({ + screen: { + title: 'Screen', + youControl: 'You control', + otherControls: 'Other controls', + agentControls: 'Bot controls', + handBack: 'Hand back', + handBackForce: 'Hand back (force)', + handBackForceHint: 'Force', + takeOver: 'Take over', + reconnect: 'Reconnect', + streamLost: 'Stream lost' + } + }) +})) +vi.mock('./screen-connection', async importActual => ({ + ...(await importActual()), + displayRequest: vi.fn(), + resolveScreenWsUrl: vi.fn(async () => 'ws://localhost/api/display/ws'), + isEventForBotScreen: () => true +})) +vi.mock('@novnc/novnc', () => ({ + default: class { + addEventListener() {} + disconnect() {} + focus() {} + } +})) + +// Real event bus, so the pane's listener path is the one under test. +// eslint-disable-next-line no-restricted-imports +import { emitGatewayEvent } from '../../contrib/events' + +import { displayRequest, viewerHash } from './screen-connection' +import { BotScreenPane } from './screen-pane' +import { $screenState } from './screen-state' + +const bot: RosterRow = { name: 'default' } +const MINTED = 'srv-viewer-0001' + +const status: DisplayStatus = { + profile: 'default', + profile_key: '/home/hermes/.hermes', + supported: true, + installed: true, + missing: [], + running: true, + pid: 42, + display: ':20', + socket: '/tmp/rfb.sock', + geometry: '1440x900', + install_command: null, + lease: { holder: 'agent', viewer_id: null, viewer_hash: null, pending_handoff: null, since: 1, reason: '' } +} + +beforeEach(() => { + $screenState.set({}) + vi.mocked(displayRequest) + .mockReset() + .mockImplementation(async (_bot, method) => + method === 'display.observe' ? { ...status, ticket: 'test-ticket', viewer_id: MINTED } : status + ) + vi.stubGlobal( + 'WebSocket', + class { + binaryType = '' + close() {} + } + ) +}) + +afterEach(() => vi.unstubAllGlobals()) + +const emitLease = (viewer_hash: string) => + act(() => + emitGatewayEvent({ + type: 'display.lease', + payload: { profile_key: status.profile_key, lease: { ...status.lease, holder: 'human', viewer_hash } } + }) + ) + +it('holds control when the lease names the hash of the server-minted viewer id, not when it names another', async () => { + const view = render() + await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe')) + await act(async () => {}) + + emitLease(await viewerHash('someone-else')) + expect(view.queryByText('You control')).toBeNull() + expect(view.getByText('Other controls')).toBeTruthy() + + emitLease(await viewerHash(MINTED)) + expect(view.getByText('You control')).toBeTruthy() + view.unmount() +}) + +it('hands back with the minted id, never a client-generated one', async () => { + const view = render() + await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe')) + await act(async () => {}) + emitLease(await viewerHash(MINTED)) + + await act(async () => { + view.getByText('Hand back').click() + }) + expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.lease.release', { viewer_id: MINTED }) + view.unmount() +}) + +it('offers a forced hand-back for a human lease this window does not hold, sending {force: true} and no viewer id', async () => { + const view = render() + await waitFor(() => expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.observe')) + await act(async () => {}) + + emitLease(await viewerHash(MINTED)) + expect(view.queryByText('Hand back (force)')).toBeNull() + + emitLease(await viewerHash('viewer-from-before-the-reload')) + await act(async () => { + view.getByText('Hand back (force)').click() + }) + expect(vi.mocked(displayRequest)).toHaveBeenCalledWith(bot, 'display.lease.release', { force: true }) + view.unmount() +}) diff --git a/apps/desktop/src/sdk/index.ts b/apps/desktop/src/sdk/index.ts index 4ce472aa508dc..39ca078d0e211 100644 --- a/apps/desktop/src/sdk/index.ts +++ b/apps/desktop/src/sdk/index.ts @@ -1501,7 +1501,15 @@ export { PanelRowMenu, PanelSectionLabel } from '@/app/overlays/panel' -export { type RouteContribution, ROUTES_AREA, SIDEBAR_NAV_AREA, type SidebarNavContribution } from '@/app/routes' +export { + type ProfileGroupHeaderContribution, + type ProfileGroupRoute, + type RouteContribution, + ROUTES_AREA, + SIDEBAR_NAV_AREA, + SIDEBAR_PROFILE_GROUP_HEADER_AREA, + type SidebarNavContribution +} from '@/app/routes' /** THE full per-toolset config panel core Settings renders — provider picker, * env vars / API keys, model catalog picker, and post-setup runners. Route- @@ -1721,6 +1729,9 @@ export const TITLEBAR_AREAS = { center: 'titleBar.center', left: 'titleBar.left' * setup.runtime_check, reconciled) — pass `host.request`. Don't hand-roll * readiness from raw RPC shapes. */ export { evaluateRuntimeReadiness, type RuntimeReadinessResult } from '@/lib/runtime-readiness' +/** A sibling WebSocket beside the route's `/api/ws` (voice PCM, Bot Screen RFB): + * same origin, same auth resolution as chat. */ +export { resolveSiblingWsUrl, type SiblingWsRoute } from '@/lib/sibling-ws-url' /** Canonical time formatting — every surface pulls from here so timestamps read * the same app-wide. For a row's AGE, bucket with `coarseElapsed` and render * the compact suffixes (`t.sidebar.row.ageMin` → "52m"), which is what the diff --git a/apps/desktop/src/store/prompts.ts b/apps/desktop/src/store/prompts.ts index 7325c4f54ed85..e08c458469716 100644 --- a/apps/desktop/src/store/prompts.ts +++ b/apps/desktop/src/store/prompts.ts @@ -98,6 +98,15 @@ interface PendingApprovalPayload { export interface SudoRequest extends KeyedPrompt { requestId: string + /** JSON-RPC method that resolves this request; the terminal tool's `sudo.respond` by default, + * `display.install.sudo.respond` for a Bot Screen package install. */ + respondMethod?: string + /** Description override so the card can say WHAT the password is for. */ + description?: string + /** Immutable origin of the request. The reply is sent to THIS socket, never to whichever + * connection happens to be in the foreground when the user presses Send — a password typed for + * host A must not travel to host B. */ + origin?: { connectionId: null | string; profile: string } } export interface SecretRequest extends KeyedPrompt { @@ -225,8 +234,10 @@ export async function replayPendingApproval(gateway: ApprovalGateway | null, ses * active-session `$*Request` views (same map, fixed key). */ export const sessionApprovalRequest = (sessionId: string | null) => computed(approval.$all, all => all[keyFor(sessionId)] ?? null) +/** A session's sudo card, else the app-level one (a Bot Screen package install is raised with no + * session: it belongs to the connection, not to a turn, so whichever chat is focused shows it). */ export const sessionSudoRequest = (sessionId: string | null) => - computed(sudo.$all, all => all[keyFor(sessionId)] ?? null) + computed(sudo.$all, all => all[keyFor(sessionId)] ?? (sessionId ? all[keyFor(null)] ?? null : null)) export const sessionSecretRequest = (sessionId: string | null) => computed(secret.$all, all => all[keyFor(sessionId)] ?? null) diff --git a/apps/desktop/src/vite-env.d.ts b/apps/desktop/src/vite-env.d.ts index 11f02fe2a0061..ad5fecb77017e 100644 --- a/apps/desktop/src/vite-env.d.ts +++ b/apps/desktop/src/vite-env.d.ts @@ -1 +1,23 @@ /// + +// @novnc/novnc ships no typings (its export is core/rfb.js); the surface the Bot Screen pane uses. +// Declared here because `apps/desktop/src/**/*.d.ts` is gitignored except for the allowlisted files, +// and an ambient `declare module` only works in a script-scoped (import-free) declaration file. +declare module '@novnc/novnc' { + export default class RFB { + constructor(target: HTMLElement, urlOrChannel: string | WebSocket | RTCDataChannel, options?: Record) + viewOnly: boolean + scaleViewport: boolean + resizeSession: boolean + focusOnClick: boolean + background: string + qualityLevel: number + compressionLevel: number + addEventListener(type: string, listener: (event: CustomEvent) => void): void + removeEventListener(type: string, listener: (event: CustomEvent) => void): void + disconnect(): void + focus(): void + blur(): void + clipboardPasteFrom(text: string): void + } +} diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 7f850a5423648..6216261a534dc 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2242,6 +2242,15 @@ def _aux(timeout, *, reasoning_effort=True, **extra): "paste_collapse_threshold_fallback": 5, "paste_collapse_char_threshold": 2000, + # Bot Desktop: a headless Xfce screen per profile on the gateway host (Linux), streamed to Hermes + # Desktop where a human can watch, take over (logins, 2FA, CAPTCHAs) and hand back. `hermes computer-use screen`. + "bot_desktop": { + "geometry": "1440x900", + # Opt-in: start the screen automatically the first time computer_use needs a display on a headless + # host. Off by default so installing TigerVNC for other reasons never yields a screen nobody asked + # for; Hermes Desktop's Screen pane offers Start and this toggle. + "auto_start": False, + }, "computer_use": { # cua-driver's upstream PostHog telemetry defaults ON; Hermes sets # CUA_DRIVER_RS_TELEMETRY_ENABLED=0 in every child env unless this is true. diff --git a/hermes_cli/dashboard_auth/ws_tickets.py b/hermes_cli/dashboard_auth/ws_tickets.py index aef1456b84e1f..65391ae239f84 100644 --- a/hermes_cli/dashboard_auth/ws_tickets.py +++ b/hermes_cli/dashboard_auth/ws_tickets.py @@ -33,11 +33,13 @@ class TicketInvalid(Exception): """Ticket missing, expired, or already consumed.""" -def mint_ticket(*, user_id: str, provider: str) -> str: +def mint_ticket(*, user_id: str, provider: str, extra: Optional[Dict[str, Any]] = None) -> str: """One-shot base64url ticket (32 random bytes) bound to this identity; ``consume_ticket`` - hands the ``info`` dict back to the WS handler.""" + hands the ``info`` dict back to the WS handler. ``extra`` rides along for routes that need + server-chosen context (the Bot Desktop bridge pins the RFB socket's profile home here so a + client can never pick another profile's screen).""" ticket = secrets.token_urlsafe(32) - info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time())} + info = {"user_id": user_id, "provider": provider, "minted_at": int(time.time()), **(extra or {})} with _lock: _tickets[ticket] = (int(time.time()) + TTL_SECONDS, info) _gc_expired_locked() diff --git a/hermes_cli/pty_bridge.py b/hermes_cli/pty_bridge.py index 90d6a61a6c338..2788eb9ecee12 100644 --- a/hermes_cli/pty_bridge.py +++ b/hermes_cli/pty_bridge.py @@ -10,13 +10,13 @@ import asyncio import errno -import fcntl +import fcntl # windows-footgun: ok — POSIX-only module by design (see docstring) import os import select import signal import struct import sys -import termios +import termios # windows-footgun: ok — POSIX-only module by design (see docstring) import time from typing import Optional, Sequence diff --git a/hermes_cli/subcommands/computer_use.py b/hermes_cli/subcommands/computer_use.py index ac0d52f4b88ad..22e0effcd6daa 100644 --- a/hermes_cli/subcommands/computer_use.py +++ b/hermes_cli/subcommands/computer_use.py @@ -174,6 +174,12 @@ def build_computer_use_parser(subparsers) -> None: "grant", help="Request the grants (opens the dialog attributed to CuaDriver)") _perms_actions = {"grant": _cu_perms_grant, "status": _cu_perms_status} + from hermes_cli.subcommands.computer_use_screen import build_screen_parser + build_screen_parser(computer_use_sub, add_json_flag) + + def _cu_screen(args): + return args.screen_func(args) + def _cu_permissions(args): handler = _perms_actions.get(getattr(args, "computer_use_perms_action", None)) if handler is not None: @@ -181,7 +187,7 @@ def _cu_permissions(args): computer_use_perms.print_help() _actions = {"install": _cu_install, "status": _cu_status, "doctor": _cu_doctor, - "permissions": _cu_permissions} + "permissions": _cu_permissions, "screen": _cu_screen} def cmd_computer_use(args): handler = _actions.get(getattr(args, "computer_use_action", None)) diff --git a/hermes_cli/subcommands/computer_use_screen.py b/hermes_cli/subcommands/computer_use_screen.py new file mode 100644 index 0000000000000..5e76b521f9918 --- /dev/null +++ b/hermes_cli/subcommands/computer_use_screen.py @@ -0,0 +1,117 @@ +"""``hermes computer-use screen`` — the Bot Desktop screen a profile's ``computer_use`` drives on a +headless Linux gateway host, viewable from Hermes Desktop. ``status`` / ``start`` / ``stop`` / +``install`` mirror the Desktop pane's controls for ops shells and cloud images.""" + +from __future__ import annotations + +import getpass +import json +import sys + + +def _screen_status(args) -> int: + from tools.bot_desktop import lease, runtime + st = runtime.status() + if bool(getattr(args, "json", False)): + print(json.dumps({**st.as_dict(), "lease": lease.get().as_dict()}, indent=2, sort_keys=True)) + return 0 if st.running else 1 + if not st.supported: + print("Bot Desktop screens run on Linux gateway hosts only (this host keeps its real display).") + return 1 + if not st.installed: + print("Bot Desktop: packages missing → " + ", ".join(st.missing)) + print(" Install: " + (st.install_command or "hermes computer-use screen install")) + return 1 + if st.running: + holder = lease.get() + who = f"human ({holder.viewer_id})" if holder.holder == "human" else "agent" + print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry}), pid {st.pid}") + print(f" control: {who} rfb socket: {st.socket}") + print(" View it: Hermes Desktop → Bots → this bot → Screen") + return 0 + print(f"Bot Desktop [{st.profile}]: installed, not running. Start: hermes computer-use screen start") + return 1 + + +def _screen_start(args) -> int: + from tools.bot_desktop import runtime + try: + st = runtime.start() + except RuntimeError as exc: + print(f"Bot Desktop: {exc}") + return 1 + print(f"Bot Desktop [{st.profile}]: running on DISPLAY {st.display} ({st.geometry})") + return 0 + + +def _screen_stop(args) -> int: + from tools.bot_desktop import runtime + if runtime.is_supported_host(): + from tools.bot_desktop import lease + lease.release() + print("Bot Desktop: stopped" if runtime.stop() else "Bot Desktop: was not running") + return 0 + + +def _screen_install(args) -> int: + from tools.bot_desktop import install, runtime + if not runtime.is_supported_host(): + print("Bot Desktop screens run on Linux gateway hosts only.") + return 1 + if not runtime.missing_binaries(): + print("Bot Desktop: packages already installed.") + return 0 + cmd = runtime.install_command() + if cmd is None: + print("Bot Desktop: no supported package manager (apt/dnf/pacman) found. Install TigerVNC (Xvnc) and " + "the Xfce core (xfwm4, xfce4-panel, xfdesktop, xfce4-settings) by hand.") + return 1 + print(f"Bot Desktop: installing → {cmd}") + if not bool(getattr(args, "yes", False)) and sys.stdin.isatty(): + answer = input("Proceed? [Y/n] ").strip().lower() + if answer not in ("", "y", "yes"): + return 1 + # Same runner as the Desktop pane's Install button: one per-profile slot, list-form spawn, sudo password + # via stdin (-S) and never on the command line. + try: + rc = install.install_packages(ask_password=lambda: getpass.getpass("[sudo] password: "), on_line=print) + except install.InstallBusy as exc: + print(f"Bot Desktop: {exc}") + return 1 + if rc != 0: + print(f"Bot Desktop: installer exited {rc}") + return rc or 1 + missing = runtime.missing_binaries() + if missing: + print("Bot Desktop: still missing " + ", ".join(missing)) + return 1 + print("Bot Desktop: ready. Start with `hermes computer-use screen start` or from Hermes Desktop.") + return 0 + + +SCREEN_ACTIONS = {"status": _screen_status, "start": _screen_start, "stop": _screen_stop, "install": _screen_install} + + +def build_screen_parser(computer_use_sub, add_json_flag) -> None: + screen = computer_use_sub.add_parser( + "screen", help="Bot Desktop: the headless screen this profile's computer_use drives (Linux)", + description="On a headless Linux gateway host Hermes gives each profile its own Xfce screen\n" + "(TigerVNC Xvnc on a private Unix socket). The agent's computer_use and headed\n" + "browser act on it; Hermes Desktop shows it live and lets a human take over for\n" + "logins, 2FA or CAPTCHAs, then hand control back.\n\n" + "`install` adds the system packages (apt/dnf/pacman); `start`/`stop` manage this\n" + "profile's screen; `status` shows display, control holder and socket.") + sub = screen.add_subparsers(dest="computer_use_screen_action") + st = sub.add_parser("status", help="Show whether this profile's screen is installed/running and who holds control") + add_json_flag(st, "Emit the status payload as JSON.") + sub.add_parser("start", help="Start this profile's screen") + sub.add_parser("stop", help="Stop this profile's screen (hands control back to the agent first)") + inst = sub.add_parser("install", help="Install TigerVNC + Xfce core via the host package manager") + inst.add_argument("-y", "--yes", action="store_true", help="Do not ask before running the package manager") + + def _cmd(args): + handler = SCREEN_ACTIONS.get(str(getattr(args, "computer_use_screen_action", None) or "")) + if handler is not None: + return handler(args) + screen.print_help() + screen.set_defaults(screen_func=_cmd) diff --git a/hermes_cli/web_routers/display.py b/hermes_cli/web_routers/display.py new file mode 100644 index 0000000000000..4ccb2721893fc --- /dev/null +++ b/hermes_cli/web_routers/display.py @@ -0,0 +1,184 @@ +"""``/api/display/ws`` — raw RFB over WebSocket for the Bot Desktop viewer. + +The Desktop renderer calls ``display.observe`` on its authenticated ``/api/ws`` connection, gets a +single-use 30 s ticket pinned to that profile's RFB socket, then opens this route with +``?display_ticket=``. No websockify, no new port: the bridge splices the profile's 0600 Unix socket +into the WebSocket as binary frames with backpressure both ways, and runs the client stream through +:class:`tools.bot_desktop.rfb_filter.RfbClientFilter` so keyboard, pointer and clipboard reach Xvnc +only from the viewer that currently holds the lease. noVNC's ``viewOnly`` is UX; this is the gate. + +A lease change closes the evicted viewer's socket with 4000 ``control-taken`` so its UI drops back to +Watch mode and reconnects. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Optional + +from fastapi import APIRouter, WebSocket, WebSocketDisconnect + +from hermes_cli.web_server_chat import _ws_request_is_allowed + +_log = logging.getLogger(__name__) +router = APIRouter() + +_READ_CHUNK = 64 * 1024 +_CLOSE_CONTROL_TAKEN = 4000 +_CLEAN_CLOSE = frozenset({1000, 1001}) +_CLOSE_DESKTOP_GONE = 4001 +_CLOSE_BAD_TICKET = 4401 +_CLOSE_NOT_ALLOWED = 4403 +_CLOSE_PROTOCOL = 1003 +_LEASE_REFRESH_S = 0.25 + + +def _should_evict(held: dict, lease, viewer_id: str) -> bool: + """A viewer that held control during this connection and lost it to ANOTHER human is kicked so its + UI repaints; a plain hand-back to the agent, pure watchers and the new holder stay connected. The + hand-back also forgets that this viewer ever held: after it, they are a plain watcher again and a + later takeover by someone else must not evict them. ``held`` is the per-connection memory.""" + from tools.bot_desktop import lease as _lease + if lease.holder != _lease.HUMAN: + held["ever"] = False + return False + if lease.viewer_id == viewer_id: + held["ever"] = True + return False + return bool(held["ever"]) + + +def _consume_display_ticket(ws: WebSocket) -> Optional[dict]: + from hermes_cli.dashboard_auth.ws_tickets import TicketInvalid, consume_ticket + ticket = ws.query_params.get("display_ticket", "") + if not ticket: + return None + try: + info = consume_ticket(ticket) + except TicketInvalid: + return None + if info.get("provider") != "bot-desktop" or not info.get("hermes_home"): + return None + return info + + +@router.websocket("/api/display/ws") +async def display_ws(ws: WebSocket) -> None: + if not _ws_request_is_allowed(ws): + await ws.close(code=_CLOSE_NOT_ALLOWED) + return + info = _consume_display_ticket(ws) + if info is None: + await ws.close(code=_CLOSE_BAD_TICKET, reason="display ticket missing, expired or used") + return + await _bridge(ws, info) + + +async def _bridge(ws: WebSocket, info: dict) -> None: + """Pump RFB bytes between the viewer socket and THIS profile's Xvnc, gated by the lease.""" + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _lease + from tools.bot_desktop.rfb_filter import RfbClientFilter + from pathlib import Path + + sock = Path(info["hermes_home"]) / "bot-desktop" / "rfb.sock" + profile_home = str(info["hermes_home"]) + profile_key = hermes_home_key(profile_home) + viewer_id = str(info.get("viewer_id") or info.get("user_id") or "viewer") + if not sock.exists(): + await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop is not running") + return + try: + reader, writer = await asyncio.open_unix_connection(str(sock)) + except OSError as exc: + _log.warning("display ws: cannot reach RFB socket %s: %s", sock, exc) + await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop socket unreachable") + return + + await ws.accept() + loop = asyncio.get_running_loop() + evicted = asyncio.Event() + held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)} + # Input gate cache: reading lease.json per client message (a stat + read on the event loop for + # every pointer move) is replaced by a decision refreshed on this process's on_change callback + # and by a file re-read at most every _LEASE_REFRESH_S, so another process's takeover still lands. + allowed = {"input": held["ever"], "at": loop.time()} + + def _refresh_allowed(lease=None) -> None: + if lease is None: + lease = _lease.get(profile_key=profile_home) + allowed["input"] = lease.holder == _lease.HUMAN and lease.viewer_id == viewer_id + allowed["at"] = loop.time() + + def _may_send_input() -> bool: + if loop.time() - allowed["at"] > _LEASE_REFRESH_S: + _refresh_allowed() + return allowed["input"] + + def _on_lease(key: str, lease) -> None: + if key != profile_key: + return + loop.call_soon_threadsafe(_refresh_allowed, lease) + if _should_evict(held, lease, viewer_id): + loop.call_soon_threadsafe(evicted.set) + unsubscribe = _lease.on_change(_on_lease) + + rfb_filter = RfbClientFilter(_may_send_input) + + viewer_closed = asyncio.Event() + + async def rfb_to_ws() -> None: + while True: + chunk = await reader.read(_READ_CHUNK) + if not chunk: + return + await ws.send_bytes(chunk) # awaiting the send is the backpressure toward Xvnc + + async def ws_to_rfb() -> None: + while True: + message = await ws.receive() + if message.get("type") == "websocket.disconnect": + # 1000/1001 = the viewer closed the window; anything else is a dropped link. + if message.get("code") in _CLEAN_CLOSE: + viewer_closed.set() + return + data = message.get("bytes") + if data is None: + await ws.close(code=_CLOSE_PROTOCOL, reason="RFB is binary") + return + try: + allowed = rfb_filter.feed(data) + except ValueError as exc: + await ws.close(code=_CLOSE_PROTOCOL, reason=str(exc)[:100]) + return + if allowed: + writer.write(allowed) + await writer.drain() # backpressure toward the browser + + async def watch_eviction() -> None: + await evicted.wait() + await ws.close(code=_CLOSE_CONTROL_TAKEN, reason="control-taken") + + tasks = [asyncio.create_task(rfb_to_ws()), asyncio.create_task(ws_to_rfb()), + asyncio.create_task(watch_eviction())] + try: + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for t in pending: + t.cancel() + for t in done: + exc = t.exception() + if exc and not isinstance(exc, (WebSocketDisconnect, ConnectionError)): + _log.debug("display ws ended: %r", exc) + finally: + unsubscribe() + writer.close() + # Closing the viewer window hands control back. A DROPPED link (laptop lid, Wi-Fi, 1006) + # keeps the human's exclusion: they may be mid-login on that screen and the agent must not + # resume into it. The Desktop reconnects into the same lease, or the human hands back. + if viewer_closed.is_set() and _lease.viewer_may_send_input(viewer_id, profile_key=profile_home): + _lease.release(viewer_id, profile_key=profile_home) + try: + await ws.close() + except Exception: # already closed by the peer or by an eviction + pass diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index d95f8182a6968..73faef43e6b35 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -932,6 +932,7 @@ def _get_dashboard_plugins(force_rescan: bool = False) -> list: status as _status_routes, actions as _actions_routes, audio as _audio_routes, + display as _display_routes, sessions as _sessions_routes, profiles as _profiles_routes, memory_providers as _memory_providers_routes, @@ -955,6 +956,7 @@ def _get_dashboard_plugins(force_rescan: bool = False) -> list: app.include_router(_status_routes.router) app.include_router(_actions_routes.router) app.include_router(_audio_routes.router) +app.include_router(_display_routes.router) app.include_router(_actions_routes.status_router) app.include_router(_sessions_routes.list_router) app.include_router(_profiles_routes.sessions_router) diff --git a/hermes_cli/web_server_chat.py b/hermes_cli/web_server_chat.py index 047bc867afb9a..7196a3254f451 100644 --- a/hermes_cli/web_server_chat.py +++ b/hermes_cli/web_server_chat.py @@ -270,7 +270,12 @@ def _stamp_identity(info) -> None: return "no_credential", "none" try: - _stamp_identity(consume_ticket(ticket)) + info = consume_ticket(ticket) + if info.get("provider") == "bot-desktop": + # A display ticket admits one RFB bridge on /api/display/ws (a watch-only + # capability handed to a screen viewer); it must not double as a login here. + raise TicketInvalid("display ticket presented as a gateway login") + _stamp_identity(info) if protocol_ticket: # Select only the stable public protocol during accept. The # ticket-bearing protocol is a credential and must never be diff --git a/package-lock.json b/package-lock.json index eaf154d450768..a24c4496ef6a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ }, "apps/bootstrap-installer": { "name": "@hermes/bootstrap-installer", - "version": "0.0.1", + "version": "0.21.1", "dependencies": { "@nous-research/ui": "0.18.2", "@tailwindcss/typography": "0.5.20", @@ -85,6 +85,7 @@ "@lezer/highlight": "1.2.3", "@nanostores/react": "1.1.0", "@nous-research/ui": "0.18.2", + "@novnc/novnc": "1.7.0", "@streamdown/code": "1.1.1", "@streamdown/math": "1.0.2", "@tabler/icons-react": "3.44.0", @@ -3498,6 +3499,12 @@ } } }, + "node_modules/@novnc/novnc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@novnc/novnc/-/novnc-1.7.0.tgz", + "integrity": "sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==", + "license": "MPL-2.0" + }, "node_modules/@npmcli/agent": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.2.2.tgz", diff --git a/pyproject.toml b/pyproject.toml index ee77c1e53a9b4..7deda29005ea2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -578,6 +578,9 @@ gateway = ["assets/**/*"] # sealed wheels with the plugin Python modules; without this declaration the # wheel contains adapters but discovery finds zero bundled plugins. plugins = ["**/plugin.yaml", "**/plugin.yml"] +# Bot Desktop starts Xvnc + Xfce through a shell launcher and seeds the wallpaper; both are read +# via Path(__file__).parent in tools/bot_desktop/runtime.py and vanish from sealed wheels otherwise. +tools = ["bot_desktop/launcher.sh", "bot_desktop/wallpaper.png"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/scripts/check-windows-footguns.py b/scripts/check-windows-footguns.py index 30100402d0ad0..d5de92b410c12 100644 --- a/scripts/check-windows-footguns.py +++ b/scripts/check-windows-footguns.py @@ -431,6 +431,27 @@ class Footgun: and _call_closes_on_line(line, m.end()) ), ), + Footgun( + name="module-level import of a POSIX-only stdlib module", + # Only unindented imports: a top-level `import fcntl` fails at import time on Windows and takes + # every importer down with it (tools.bot_desktop.lease took computer_use down on native + # Windows). Indented imports inside a function or a try/except ImportError are the fix shape. + pattern=re.compile( + r"^(?:import\s+(?:fcntl|pwd|grp|termios|resource|pty|tty)\b" + r"|from\s+(?:fcntl|pwd|grp|termios|resource|pty|tty)\s+import\b)" + ), + message=( + "fcntl/pwd/grp/termios/resource/pty/tty do not exist on Windows; a module-level import " + "raises ModuleNotFoundError and breaks every module that imports this one." + ), + fix=( + "Import lazily inside the function that needs it, or\n" + "try:\n" + " import fcntl\n" + "except ImportError:\n" + " fcntl = None # and take the Windows path when None" + ), + ), ] diff --git a/scripts/profile-tui.py b/scripts/profile-tui.py index 4d86aa057bfa9..a59b76659d822 100755 --- a/scripts/profile-tui.py +++ b/scripts/profile-tui.py @@ -26,7 +26,7 @@ import argparse import json import os -import pty +import pty # windows-footgun: ok — dev profiling script, POSIX pty by design import select import signal import sqlite3 diff --git a/tests/hermes_cli/test_bot_desktop_screen_stop.py b/tests/hermes_cli/test_bot_desktop_screen_stop.py new file mode 100644 index 0000000000000..33f661aed5077 --- /dev/null +++ b/tests/hermes_cli/test_bot_desktop_screen_stop.py @@ -0,0 +1,19 @@ +"""The documented CLI stop is also recovery for a disconnected viewer's lease.""" + +import argparse + +from hermes_cli.subcommands.computer_use_screen import build_screen_parser +import pytest + + +@pytest.mark.linux_only +def test_screen_stop_hands_back_even_when_the_desktop_has_already_exited(): + from tools.bot_desktop import lease + + parser = argparse.ArgumentParser() + build_screen_parser(parser.add_subparsers(), lambda sub, help_text: sub.add_argument('--json', action='store_true')) + lease.acquire('disconnected-viewer') + args = parser.parse_args(['screen', 'stop']) + assert args.screen_func(args) == 0 + assert lease.get().holder == lease.AGENT + assert lease.wait_for_release(timeout=0) diff --git a/tests/hermes_cli/test_computer_use_screen_install.py b/tests/hermes_cli/test_computer_use_screen_install.py new file mode 100644 index 0000000000000..a0199600801c2 --- /dev/null +++ b/tests/hermes_cli/test_computer_use_screen_install.py @@ -0,0 +1,29 @@ +"""`hermes computer-use screen install` shares the Desktop pane's installer (one lock, list-form spawn).""" + +from __future__ import annotations + +import argparse +import subprocess + +from hermes_cli.subcommands.computer_use_screen import build_screen_parser +from tools.bot_desktop import install, runtime + + +def test_cli_install_goes_through_the_shared_installer(monkeypatch): + monkeypatch.setattr(runtime, "is_supported_host", lambda: True) + monkeypatch.setattr(runtime, "missing_binaries", lambda: ["Xvnc"]) + monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y tigervnc-standalone-server") + monkeypatch.setattr(subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(AssertionError("shell install spawned"))) + calls = [] + + def fake_install(*, ask_password, on_line, **kw): + calls.append((callable(ask_password), callable(on_line))) + return 0 + + monkeypatch.setattr(install, "install_packages", fake_install) + monkeypatch.setattr(runtime, "missing_binaries", lambda: ["Xvnc"] if not calls else []) + parser = argparse.ArgumentParser() + build_screen_parser(parser.add_subparsers(), lambda sub, help_text: sub.add_argument("--json", action="store_true")) + args = parser.parse_args(["screen", "install", "-y"]) + assert args.screen_func(args) == 0 + assert calls == [(True, True)] diff --git a/tests/hermes_cli/test_display_ws_drop_keeps_lease.py b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py new file mode 100644 index 0000000000000..2bc98c15dba05 --- /dev/null +++ b/tests/hermes_cli/test_display_ws_drop_keeps_lease.py @@ -0,0 +1,130 @@ +"""A dropped viewer link (1006: lid closed, Wi-Fi) must NOT hand the screen back to the agent — the +human may be mid-login on it. Only a clean close (1000/1001) releases.""" + +from __future__ import annotations + +import asyncio +import os +import tempfile + +import pytest + +from hermes_cli.web_routers import display +from tools.bot_desktop import lease + + +class _Ws: + """Just enough of a Starlette WebSocket: one disconnect message with the given close code.""" + + def __init__(self, close_code: int): + self._code = close_code + self.closed = False + + async def accept(self): + pass + + async def receive(self): + await asyncio.sleep(0.05) + return {"type": "websocket.disconnect", "code": self._code} + + async def send_bytes(self, data): + pass + + async def close(self, code=1000, reason=""): + self.closed = True + + +async def _bridge_once(close_code: int, home: str) -> lease.Lease: + sock_dir = os.path.join(home, "bot-desktop") + os.makedirs(sock_dir, exist_ok=True) + sock = os.path.join(sock_dir, "rfb.sock") + + async def _xvnc(reader, writer): # a silent framebuffer + await asyncio.sleep(1) + writer.close() + + server = await asyncio.start_unix_server(_xvnc, path=sock) + try: + info = {"hermes_home": home, "viewer_id": "desk-1"} + await display._bridge(_Ws(close_code), info) + finally: + server.close() + return lease.get(profile_key=home) + + +# 1005 (no status code) is what noVNC's code-less socket.close() AND some proxies produce on a drop, so +# the server keeps the lease; the Desktop sends an explicit 1000 when the pane is closed on purpose. +@pytest.mark.parametrize(("close_code", "human_keeps_control"), [(1006, True), (1005, True), (1000, False)]) +def test_only_a_clean_viewer_close_hands_the_screen_back(monkeypatch, close_code, human_keeps_control): + lease._reset_for_tests() + with tempfile.TemporaryDirectory() as home: + lease.acquire("desk-1", profile_key=home) + after = asyncio.run(_bridge_once(close_code, home)) + lease._reset_for_tests() + assert (after.holder == lease.HUMAN) is human_keeps_control, after + + +def test_ex_holder_who_handed_back_is_not_evicted_by_a_later_takeover(): + """desk-1 holds, hands back to the agent, then desk-2 takes over: desk-1 is a plain watcher again + and must stay connected; only a takeover WHILE desk-1 held (or believed it held) kicks it.""" + held = {"ever": False} + assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-1"), "desk-1") is False + assert display._should_evict(held, lease.Lease(holder=lease.AGENT), "desk-1") is False + assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-2"), "desk-1") is False + + held = {"ever": False} + display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-1"), "desk-1") + assert display._should_evict(held, lease.Lease(holder=lease.HUMAN, viewer_id="desk-2"), "desk-1") is True + + +class _OpenWs(_Ws): + """Stays open until ``finish`` is set, then reports a clean close.""" + + def __init__(self): + super().__init__(1000) + self.finish = asyncio.Event() + + async def receive(self): + await self.finish.wait() + return {"type": "websocket.disconnect", "code": self._code} + + +def test_a_takeover_made_by_another_process_stops_input_within_the_refresh_interval(monkeypatch): + """The bridge caches the input decision instead of reading lease.json per message; a takeover + written by ANOTHER process (no in-process listener fires) must still be seen quickly.""" + from tools.bot_desktop import rfb_filter + captured = {} + + class _Filter(rfb_filter.RfbClientFilter): + def __init__(self, allow_input): + super().__init__(allow_input) + captured["allow"] = allow_input + monkeypatch.setattr(rfb_filter, "RfbClientFilter", _Filter) + + async def _run(home: str) -> float: + sock_dir = os.path.join(home, "bot-desktop") + os.makedirs(sock_dir, exist_ok=True) + server = await asyncio.start_unix_server(lambda r, w: None, path=os.path.join(sock_dir, "rfb.sock")) + ws = _OpenWs() + task = asyncio.create_task(display._bridge(ws, {"hermes_home": home, "viewer_id": "desk-1"})) + try: + while "allow" not in captured: + await asyncio.sleep(0.01) + assert captured["allow"]() is True + # Another process takes over: the file changes, no listener in this process is told. + lease._write(lease._path(home), lease.Lease(holder=lease.HUMAN, viewer_id="desk-2", epoch=2)) + t0 = asyncio.get_running_loop().time() + while captured["allow"]() and asyncio.get_running_loop().time() - t0 < 2.0: + await asyncio.sleep(0.02) + return asyncio.get_running_loop().time() - t0 + finally: + ws.finish.set() + await task + server.close() + + lease._reset_for_tests() + with tempfile.TemporaryDirectory() as home: + lease.acquire("desk-1", profile_key=home) + elapsed = asyncio.run(_run(home)) + lease._reset_for_tests() + assert elapsed < 0.5, elapsed diff --git a/tests/hermes_cli/test_display_ws_ticket.py b/tests/hermes_cli/test_display_ws_ticket.py new file mode 100644 index 0000000000000..fc072596b15fe --- /dev/null +++ b/tests/hermes_cli/test_display_ws_ticket.py @@ -0,0 +1,27 @@ +"""The display bridge admits only a display ticket minted for THIS profile's socket: a gateway ticket, +an expired ticket, or one for another provider is refused before any socket is dialled.""" + +from __future__ import annotations + +from hermes_cli.dashboard_auth import ws_tickets +from hermes_cli.web_routers import display + + +class _Ws: + def __init__(self, **params): + self.query_params = params + + +def test_display_ticket_must_be_a_bot_desktop_ticket_pinned_to_a_profile_home(monkeypatch): + ws_tickets._reset_for_tests() + gateway_ticket = ws_tickets.mint_ticket(user_id="u", provider="google") + assert display._consume_display_ticket(_Ws(display_ticket=gateway_ticket)) is None + + unpinned = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop") + assert display._consume_display_ticket(_Ws(display_ticket=unpinned)) is None + + good = ws_tickets.mint_ticket(user_id="display:v", provider="bot-desktop", + extra={"hermes_home": "/srv/hermes/bot-a", "viewer_id": "v"}) + info = display._consume_display_ticket(_Ws(display_ticket=good)) + assert info and info["hermes_home"] == "/srv/hermes/bot-a" and info["viewer_id"] == "v" + assert display._consume_display_ticket(_Ws(display_ticket=good)) is None, "single use" diff --git a/tests/hermes_cli/test_ws_auth_display_ticket.py b/tests/hermes_cli/test_ws_auth_display_ticket.py new file mode 100644 index 0000000000000..626f2c106c38c --- /dev/null +++ b/tests/hermes_cli/test_ws_auth_display_ticket.py @@ -0,0 +1,37 @@ +"""A Bot Desktop display ticket admits one RFB bridge on ``/api/display/ws`` — it must never pass as a +full login on ``/api/ws`` (the reverse of ``test_display_ws_ticket``, which refuses gateway tickets there).""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from hermes_cli import web_server +import hermes_cli.web_server_chat as _web_server_chat +from hermes_cli.dashboard_auth.ws_tickets import _reset_for_tests, mint_ticket + + +@pytest.fixture +def gated_state(): + _reset_for_tests() + prev = getattr(web_server.app.state, "auth_required", None) + web_server.app.state.auth_required = True + yield + web_server.app.state.auth_required = prev + _reset_for_tests() + + +def _ws(ticket: str): + return SimpleNamespace( + query_params={"ticket": ticket}, headers={}, + client=SimpleNamespace(host="203.0.113.9"), url=SimpleNamespace(path="/api/ws")) + + +def test_display_ticket_is_refused_as_a_gateway_login(gated_state): + ticket = mint_ticket(user_id="display:v", provider="bot-desktop", + extra={"hermes_home": "/srv/hermes/bot-a", "viewer_id": "v"}) + ws = _ws(ticket) + reason, _credential = _web_server_chat._ws_auth_reason(ws) + assert reason == "ticket_invalid" + assert not hasattr(ws, "_hermes_auth_identity") diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index f5c2c431fd296..34811b799b72f 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -35,6 +35,24 @@ def _no_host_browser_use_cli(): yield +@pytest.fixture(autouse=True) +def _no_host_bot_desktop_autostart(): + """Keep the host's TigerVNC/Xfce install out of tests. + + ``computer_use`` auto-starts the profile's Bot Desktop on a headless Linux + host with the packages installed, so a developer box that has them would + launch a real Xvnc + Xfce session per test. Pin the binaries to "missing"; + tests that exercise the desktop path monkeypatch ``runtime`` themselves. + """ + try: + from tools.bot_desktop import runtime as bd_runtime + except Exception: + yield + return + with patch.object(bd_runtime, "missing_binaries", lambda: ["Xvnc"]): + yield + + @pytest.fixture(autouse=True) def _materialize_mcp_sdk_symbols(): """Materialize the lazily-imported MCP SDK before each tools test. diff --git a/tests/tools/test_bot_desktop_browser.py b/tests/tools/test_bot_desktop_browser.py new file mode 100644 index 0000000000000..451702e2c4f32 --- /dev/null +++ b/tests/tools/test_bot_desktop_browser.py @@ -0,0 +1,91 @@ +"""The dock's Browser and agent-browser resolve to one identity: same executable, same user-data-dir — +and when a human opened that browser first, the agent attaches to it instead of launching a second one +(Chromium's profile singleton would forward the launch and kill it without a DevTools endpoint).""" + +from __future__ import annotations + +import os +import socket + +from tools.bot_desktop import browser, runtime + + +def test_dock_and_agent_share_browser_identity(tmp_path, monkeypatch): + exe = tmp_path / "chrome" + exe.write_text("#!/bin/sh\n", encoding="utf-8") + exe.chmod(0o755) + monkeypatch.setenv("AGENT_BROWSER_EXECUTABLE_PATH", str(exe)) + monkeypatch.delenv("AGENT_BROWSER_PROFILE", raising=False) + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bot-desktop") + + dock_exe, dock_profile = browser.dock_launch() + agent_env = browser.env_for_agent({}) + + assert dock_exe == agent_env["AGENT_BROWSER_EXECUTABLE_PATH"] == str(exe) + assert dock_profile == agent_env["AGENT_BROWSER_PROFILE"] == str(tmp_path / "bot-desktop" / "browser-profile") + + +def test_user_pinned_profile_wins(tmp_path, monkeypatch): + monkeypatch.setenv("AGENT_BROWSER_PROFILE", str(tmp_path / "mine")) + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path / "bot-desktop") + assert browser.profile_dir() == tmp_path / "mine" + + +def test_dock_browser_advertises_a_devtools_port(): + """A human-started instance must be attachable, or the agent can never drive it afterwards.""" + assert "--remote-debugging-port=" in browser.dock_command("/opt/chrome", "/p/dir").split()[2] + + +def _fake_running_instance(user_data_dir, pid: int, port: int) -> None: + (user_data_dir / "DevToolsActivePort").write_text(f"{port}\n/devtools/browser/abc\n", encoding="utf-8") + os.symlink(f"host-{pid}", user_data_dir / "SingletonLock") + + +def test_running_instance_port_requires_live_pid_and_open_port(tmp_path): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + port = listener.getsockname()[1] + try: + _fake_running_instance(tmp_path, os.getpid(), port) + assert browser.running_instance_cdp_port(str(tmp_path)) == port + + # Both files outlive a closed Chromium: a dead pid must not be trusted. + os.unlink(tmp_path / "SingletonLock") + os.symlink("host-2147483000", tmp_path / "SingletonLock") + assert browser.running_instance_cdp_port(str(tmp_path)) is None + finally: + listener.close() + # Live pid, port no longer accepting: still not attachable. + os.unlink(tmp_path / "SingletonLock") + os.symlink(f"host-{os.getpid()}", tmp_path / "SingletonLock") + assert browser.running_instance_cdp_port(str(tmp_path)) is None + assert browser.running_instance_cdp_port(str(tmp_path / "missing")) is None + + +def test_agent_attaches_to_human_started_browser(monkeypatch): + """With a live dock instance on the shared profile the local argv carries ``--cdp ``; without one + it stays a plain ``--session`` launch.""" + from tools import browser_tool_session as session + + monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":37"}) + monkeypatch.setattr(session._cloud, "_get_browser_engine", lambda: "auto") + monkeypatch.setattr(session._cloud, "_is_headed_mode", lambda: False) + monkeypatch.setattr(session, "_agent_browser_argv", lambda cmd: [cmd]) + argvs: list = [] + + def spawn(task_id, session_info, cmd_parts, *rest): + argvs.append(cmd_parts) + return {"success": True} + + monkeypatch.setattr(session, "_spawn_and_collect", spawn) + monkeypatch.setattr(session._lp, "_lightpanda_fallback_reason", lambda *a: None) + info = {"session_name": "h_abc", "cdp_url": None, "features": {"local": True}} + + monkeypatch.setattr(browser, "running_instance_cdp_port", lambda d, **kw: 41234) + session._run_browser_command_unfenced("t", "open", ["https://x"], 10, None, "agent-browser", info) + assert argvs[-1][:5] == ["agent-browser", "--session", "h_abc", "--cdp", "41234"] + + monkeypatch.setattr(browser, "running_instance_cdp_port", lambda d, **kw: None) + session._run_browser_command_unfenced("t", "open", ["https://x"], 10, None, "agent-browser", info) + assert "--cdp" not in argvs[-1] and argvs[-1][:3] == ["agent-browser", "--session", "h_abc"] diff --git a/tests/tools/test_bot_desktop_browser_fence.py b/tests/tools/test_bot_desktop_browser_fence.py new file mode 100644 index 0000000000000..743deec2de8be --- /dev/null +++ b/tests/tools/test_bot_desktop_browser_fence.py @@ -0,0 +1,78 @@ +"""The bot's browser tools obey the screen lease: while a human holds the shared browser, nothing is +dispatched, and a command whose run crossed a takeover loses its result.""" + +from __future__ import annotations + +import json + +import pytest + +from tools.bot_desktop import lease, runtime + + +@pytest.fixture(autouse=True) +def _fresh(monkeypatch): + lease._reset_for_tests() + monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":37"}) + yield + lease._reset_for_tests() + + +def _wire(monkeypatch, commands): + from tools import browser_tool as browser + from tools import browser_tool_session as session + + monkeypatch.delenv("AGENT_BROWSER_PROFILE", raising=False) + monkeypatch.setattr(browser, "_is_camofox_mode", lambda: False) + monkeypatch.setattr(browser, "_blocked_private_page_action", lambda *a: None) + monkeypatch.setattr(session, "_browser_command_preflight", lambda: {"browser_cmd": "agent-browser"}) + monkeypatch.setattr(session, "_get_session_info", lambda *a: {"session_name": "review", "cdp_url": None, "features": {"local": True}}) + monkeypatch.setattr(session._cloud, "_get_browser_engine", lambda: "chrome") + monkeypatch.setattr(session._cloud, "_is_headed_mode", lambda: True) + + def spawn(*args): + commands.append(args[2]) + return {"success": True, "data": {"secret": "WHAT-THE-HUMAN-TYPED"}} + + monkeypatch.setattr(session, "_spawn_and_collect", spawn) + return browser, session + + +def test_browser_click_is_fenced_while_human_controls_shared_browser(monkeypatch): + commands: list = [] + browser, _ = _wire(monkeypatch, commands) + lease.acquire("human-viewer") + result = json.loads(browser.browser_click("e1", task_id="review")) + assert commands == [], f"human holds the lease, yet a browser command was dispatched: {commands}" + assert result.get("code") == "human_has_control" + + +def test_browser_result_crossing_a_takeover_is_discarded(monkeypatch): + commands: list = [] + browser, session = _wire(monkeypatch, commands) + + def spawn_then_takeover(*args): + lease.acquire("human-viewer") + lease.release("human-viewer") # a full cycle, control is back — the frame is still theirs + return {"success": True, "data": {"secret": "WHAT-THE-HUMAN-TYPED"}} + + monkeypatch.setattr(session, "_spawn_and_collect", spawn_then_takeover) + result = browser.browser_click("e1", task_id="review") + assert "WHAT-THE-HUMAN-TYPED" not in result + + +def test_real_profile_local_browser_is_fenced_by_provenance_even_without_a_live_display(monkeypatch): + """A real-profile session attaches over a loopback cdp_url but is launched with the Bot Desktop + DISPLAY, so it IS the human's browser: the fence keys on the ``local`` feature, not on the + transport. And a stranded human lease with the screen already down must still fence (computer_use + does), not silently unfence the browser.""" + commands: list = [] + browser, session = _wire(monkeypatch, commands) + monkeypatch.setattr(session, "_get_session_info", lambda *a: { + "session_name": "rp_1", "cdp_url": "ws://127.0.0.1:9222/devtools/browser/x", + "features": {"local": True, "real_profile": True}}) + monkeypatch.setattr(runtime, "published_env", lambda: {}) + lease.acquire("human-viewer") + result = json.loads(browser.browser_click("e1", task_id="review")) + assert commands == [], f"human holds the lease, yet a real-profile browser command was dispatched: {commands}" + assert result.get("code") == "human_has_control" diff --git a/tests/tools/test_bot_desktop_install.py b/tests/tools/test_bot_desktop_install.py new file mode 100644 index 0000000000000..a3e5392a66a28 --- /dev/null +++ b/tests/tools/test_bot_desktop_install.py @@ -0,0 +1,99 @@ +"""Bot Desktop package install: sudo hand-off and single-flight invariants.""" + +from __future__ import annotations + +import threading + +import pytest + +from tools.bot_desktop import install, runtime + + +@pytest.fixture(autouse=True) +def _isolated_host(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(install, "_sudo_nopasswd", lambda: False) + monkeypatch.setattr(runtime, "is_supported_host", lambda: True) + monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y tigervnc-standalone-server") + yield + install._running.clear() + + +def test_empty_password_cancels_without_spawning(monkeypatch): + monkeypatch.setattr(install.subprocess, "Popen", lambda *a, **k: pytest.fail("package manager spawned")) + lines: list[str] = [] + code = install.install_packages(ask_password=lambda: "", on_line=lines.append) + assert code == -1 + assert any("cancelled" in line for line in lines) + + +def test_second_install_for_same_profile_is_refused(monkeypatch): + gate = threading.Event() + entered = threading.Event() + + def slow_run(cmd, *, ask_password, on_line, timeout_seconds): + entered.set() + gate.wait(5) + return 0 + + monkeypatch.setattr(install, "_run", slow_run) + worker = threading.Thread(target=install.install_packages, kwargs={"ask_password": lambda: "pw", "on_line": lambda _l: None}) + worker.start() + assert entered.wait(5) + with pytest.raises(install.InstallBusy): + install.install_packages(ask_password=lambda: "pw", on_line=lambda _l: None) + gate.set() + worker.join(5) + install.assert_not_running() # slot released once the run finishes + + +def test_claim_is_atomic_and_refuses_a_second_claim(): + """The gateway claims BEFORE spawning its worker; a second Install click must fail at claim time, not + pass a read-only check and race the worker for the slot.""" + key = install.claim() + with pytest.raises(install.InstallBusy): + install.claim() + with pytest.raises(install.InstallBusy): + install.install_packages(ask_password=lambda: "pw", on_line=lambda _l: None) + install.release(key) + install.claim() # free again + install.release(key) + + +@pytest.mark.linux_only +def test_timeout_kills_the_package_managers_whole_process_group(monkeypatch): + """sudo forks the package manager into the same (new) session; killing sudo alone leaves apt/dnf + holding the dpkg lock as root. The timeout must take the group.""" + import subprocess + import time + + monkeypatch.setattr(install, "_sudo_nopasswd", lambda: True) + # stand-in for `sudo apt-get ...`: a parent that spawns a child and waits, both in the new session + fake = ["sudo"] + real_popen = subprocess.Popen + + def popen(argv, **kw): + if argv[:1] != fake: + return real_popen(argv, **kw) + return real_popen(["bash", "-c", "sleep 30 >/dev/null 2>&1 & echo child $!; wait"], **kw) + + monkeypatch.setattr(install.subprocess, "Popen", popen) + lines: list[str] = [] + code = install._run("sudo apt-get install -y x", ask_password=lambda: "", on_line=lines.append, timeout_seconds=0.5) + assert code != 0 + child = next(int(line.split()[1]) for line in lines if line.startswith("child ")) + from pathlib import Path + + def gone() -> bool: # /proc-based: a reparented orphan sits outside our subtree, where os.kill(pid, 0) is guarded + try: + return "Z" in (Path(f"/proc/{child}/stat").read_text().rsplit(")", 1)[1].split() or ["Z"])[0] + except OSError: + return True + + for _ in range(50): # the child must die with the group, not linger reparented to init + if gone(): + break + time.sleep(0.1) + else: + subprocess.run(["kill", "-9", str(child)], check=False) + pytest.fail("grandchild survived the install timeout") diff --git a/tests/tools/test_bot_desktop_launcher_seed.py b/tests/tools/test_bot_desktop_launcher_seed.py new file mode 100644 index 0000000000000..261d9d00558dc --- /dev/null +++ b/tests/tools/test_bot_desktop_launcher_seed.py @@ -0,0 +1,64 @@ +"""Bot Desktop launcher seeds: the dock only points at programs that exist, the look is applied.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +LAUNCHER = Path(__file__).resolve().parents[2] / "tools" / "bot_desktop" / "launcher.sh" +pytestmark = pytest.mark.linux_only + + +def _seed(tmp_path: Path, fake_bins: list[str], browser_exec: str = "") -> Path: + bindir = tmp_path / "bin" + bindir.mkdir() + for name in fake_bins: + exe = bindir / name + exe.write_text("#!/bin/sh\n", encoding="utf-8") + exe.chmod(0o755) + # The script's own tooling (mkdir, sed, cat, awk...) symlinked in, so PATH need not contain the + # host's /usr/bin where a real chrome/thunar would leak into the dock under test. + for tool in ("mkdir", "sed", "cat", "printf", "dirname", "bash", "sh", "rm", "ln", "touch", "chmod", "xauth", "od", "tr", "awk"): + real = shutil.which(tool) + if real and not (bindir / tool).exists(): + (bindir / tool).symlink_to(real) + cfg = tmp_path / "xdg" + env = { + "PATH": str(bindir), + "HOME": str(tmp_path), + "HERMES_BD_PROFILE": "t", "HERMES_BD_DISPLAY_NUM": "99", + "HERMES_BD_SOCKET": str(tmp_path / "rfb.sock"), "HERMES_BD_XAUTH": str(tmp_path / "Xauthority"), + "HERMES_BD_ENV_FILE": str(tmp_path / "env"), "HERMES_BD_CONFIG_HOME": str(cfg), + "HERMES_BD_SEED_ONLY": "1", + **({"HERMES_BD_BROWSER_EXEC": browser_exec} if browser_exec else {}), + } + subprocess.run(["bash", str(LAUNCHER)], env=env, check=True, stdin=subprocess.DEVNULL, capture_output=True, timeout=30) + return cfg + + +def test_dock_lists_only_programs_present_on_path(tmp_path): + chrome = tmp_path / "bin" / "chrome" # the browser is the one runtime.py resolved, never a PATH scan + cfg = _seed(tmp_path, ["xfce4-terminal", "chrome", "firefox"], browser_exec=f"{chrome} --user-data-dir={tmp_path}/bp") + panel = ET.parse(cfg / "xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml") # well-formed or this raises + launcher_ids = [str(p.get("name")) for p in panel.iter("property") if p.get("value") == "launcher"] + execs = sorted( + line.split("=", 1)[1] + for pid in launcher_ids + for line in (cfg / "xfce4/panel" / pid.replace("plugin-", "launcher-") / "hermes.desktop").read_text(encoding="utf-8").splitlines() + if line.startswith("Exec=") + ) + assert execs == [f"{chrome} --user-data-dir={tmp_path}/bp", "xfce4-terminal"] + + +def test_look_is_seeded_with_wallpaper_and_theme(tmp_path): + cfg = _seed(tmp_path, ["xfce4-terminal"]) + desktop = (cfg / "xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml").read_text(encoding="utf-8") + xsettings = (cfg / "xfce4/xfconf/xfce-perchannel-xml/xsettings.xml").read_text(encoding="utf-8") + assert str(LAUNCHER.with_name("wallpaper.png")) in desktop + assert "PLACEHOLDER" not in desktop + xsettings + assert os.path.isfile(LAUNCHER.with_name("wallpaper.png")) diff --git a/tests/tools/test_bot_desktop_launcher_xvnc.py b/tests/tools/test_bot_desktop_launcher_xvnc.py new file mode 100644 index 0000000000000..c2861796c2491 --- /dev/null +++ b/tests/tools/test_bot_desktop_launcher_xvnc.py @@ -0,0 +1,41 @@ +"""The Bot Desktop's Xvnc is started with the RFB options the lease design relies on.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +LAUNCHER = Path(__file__).resolve().parents[2] / "tools" / "bot_desktop" / "launcher.sh" +pytestmark = pytest.mark.linux_only + + +def test_xvnc_never_sends_the_holders_clipboard_to_watchers(tmp_path): + """Whoever holds control may paste INTO the screen (AcceptCutText), but the screen's clipboard must + not be pushed to every connected viewer (SendCutText off): watchers are not the holder.""" + bindir = tmp_path / "bin" + bindir.mkdir() + argv_log = tmp_path / "xvnc-argv" + (bindir / "Xvnc").write_text(f'#!/bin/sh\nprintf "%s\\n" "$@" > "{argv_log}"\nexec sleep 3\n', encoding="utf-8") + for stub in ("xdpyinfo", "setxkbmap", "xsetroot", "xset", "dbus-run-session"): + (bindir / stub).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + for exe in bindir.iterdir(): + exe.chmod(0o755) + for tool in ("mkdir", "sed", "cat", "printf", "dirname", "bash", "sh", "rm", "ln", "touch", "chmod", "xauth", "od", "tr", "awk", "seq", "sleep", "kill"): + real = shutil.which(tool) + if real and not (bindir / tool).exists(): + (bindir / tool).symlink_to(real) + env = { + "PATH": str(bindir), "HOME": str(tmp_path), + "HERMES_BD_PROFILE": "t", "HERMES_BD_DISPLAY_NUM": "99", + "HERMES_BD_SOCKET": str(tmp_path / "rfb.sock"), "HERMES_BD_XAUTH": str(tmp_path / "Xauthority"), + "HERMES_BD_ENV_FILE": str(tmp_path / "env"), "HERMES_BD_CONFIG_HOME": str(tmp_path / "xdg"), + } + subprocess.run(["bash", str(LAUNCHER)], env=env, check=True, stdin=subprocess.DEVNULL, capture_output=True, timeout=30) + argv = argv_log.read_text(encoding="utf-8").split("\n") + assert "-SendCutText=0" in argv, argv + assert not any(a.startswith("-AcceptCutText") for a in argv), "paste into the screen must keep working" + assert not os.path.exists(tmp_path / "rfb.sock") # stub never bound it; nothing leaked diff --git a/tests/tools/test_bot_desktop_lease.py b/tests/tools/test_bot_desktop_lease.py new file mode 100644 index 0000000000000..50f948f46ad57 --- /dev/null +++ b/tests/tools/test_bot_desktop_lease.py @@ -0,0 +1,152 @@ +"""Bot Desktop invariants: the byte-level RFB input gate follows the lease, and computer_use refuses +every action (capture included) while a human holds the screen.""" + +from __future__ import annotations + +import json + +import pytest + +from tools.bot_desktop import lease +from tools.bot_desktop.rfb_filter import RfbClientFilter + +_HANDSHAKE = b"RFB 003.008\n" + b"\x01" + b"\x00" +_KEY = b"\x04\x01\x00\x00\x00\x00\x00\x61" # KeyEvent 'a' down +# QEMU Extended KeyEvent (type 255, sub 0): what noVNC sends once Xvnc advertises the pseudo-encoding. +_QEMU_KEY = bytes([255, 0, 0, 1]) + (0x65).to_bytes(4, "big") + (0x12).to_bytes(4, "big") +_POINTER = b"\x05\x01\x00\x10\x00\x10" # PointerEvent, button 1 +_CUT = b"\x06\x00\x00\x00\x00\x00\x00\x02hi" # ClientCutText "hi" +_FBUR = b"\x03\x00" + b"\x00" * 8 # FramebufferUpdateRequest +_SETENC = b"\x02\x00\x00\x02" + b"\x00\x00\x00\x07" + b"\xff\xff\xff\x21" # SetEncodings x2 + + +@pytest.fixture(autouse=True) +def _fresh_lease(): + lease._reset_for_tests() + yield + lease._reset_for_tests() + + +def test_rfb_filter_forwards_input_only_from_the_lease_holder_across_arbitrary_chunking(): + f = RfbClientFilter(lambda: lease.viewer_may_send_input("v1")) + head = f.feed(_HANDSHAKE) + assert head[-1:] == b"\x01", "ClientInit is forced shared so a viewer never kicks the agent's watcher" + + # Agent holds: read-only messages pass, input is dropped, even when split byte by byte. + stream = _KEY + _FBUR + _POINTER + _SETENC + _CUT + _QEMU_KEY + out = b"".join(f.feed(stream[i:i + 1]) for i in range(len(stream))) + assert out == _FBUR + _SETENC + + lease.acquire("v1") + assert f.feed(_KEY + _POINTER + _QEMU_KEY) == _KEY + _POINTER + _QEMU_KEY + + lease.acquire("v2") # last writer wins: v1 is evicted from input on the very next message + assert f.feed(_KEY) == b"" + assert lease.release("v1").holder == lease.HUMAN, "a stale viewer's release must not yank control from v2" + assert lease.release("v2").holder == lease.AGENT + + +def test_computer_use_refuses_every_action_while_a_human_holds_the_screen(monkeypatch): + from tools.computer_use import tool + + calls = [] + monkeypatch.setattr(tool, "_get_backend", lambda session_id="": calls.append(session_id) or object()) + lease.acquire("human") + for action in ("capture", "click", "type", "list_windows"): + res = json.loads(tool.handle_computer_use({"action": action, "text": "pw"})) + assert res["code"] == "human_has_control", action + assert calls == [], "the driver is never touched while the human may be typing a credential" + + # Handoff round trip: the agent asks, the human takes over and hands back, the agent is unblocked. + asked = json.loads(tool.handle_computer_use({"action": "request_handoff", "reason": "log in"})) + assert asked["ok"] and asked["state"]["pending_handoff"] == "log in" + lease.acquire("human", reason="log in") + assert lease.get().pending_handoff is None + lease.release("human") + done = json.loads(tool.handle_computer_use({"action": "wait_for_human", "seconds": 1})) + assert done["ok"] and done["state"]["holder"] == lease.AGENT + + +def test_lease_authority_is_shared_across_processes(tmp_path): + """The gateway that streams the screen and the process running the agent are different processes; + a human takeover in one must refuse actions in the other.""" + import os + import subprocess + import sys + + lease.acquire("desktop-viewer") + probe = ("import sys; sys.path.insert(0, %r)\n" + "from tools.bot_desktop import lease\n" + "try:\n lease.assert_agent_may_act(); print('AGENT')\n" + "except lease.HumanHasControl:\n print('HUMAN')\n" + "lease.release('desktop-viewer')\n") % os.getcwd() + out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, encoding="utf-8", timeout=30, + stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": os.environ["HERMES_HOME"]}) + assert out.stdout.strip() == "HUMAN", out.stderr + assert lease.get().holder == lease.AGENT, "the other process's release is visible here" + + +def test_takeover_during_an_admitted_action_discards_its_result(monkeypatch): + """Approval / backend start-up can take seconds; a human who takes over meanwhile must not have + their keystrokes captured by an action admitted before they did.""" + from tools.computer_use import tool + + monkeypatch.setattr(tool, "_get_backend", lambda session_id="": object()) + + def _dispatch_then_takeover(backend, action, args, **_): + lease.acquire("human") # a whole take-over / hand-back cycle inside the driver call: + lease.release("human") # control is back, but the frame is still the human's turn + return json.dumps({"ok": True, "action": action, "png_b64": "SECRET"}) + + monkeypatch.setattr(tool, "_dispatch", _dispatch_then_takeover) + res = json.loads(tool.handle_computer_use({"action": "capture"})) + assert res["code"] == "human_has_control" and "SECRET" not in json.dumps(res) + + +def test_unreadable_lease_file_fails_closed_and_takeover_keeps_the_agents_reason(tmp_path): + """Missing file = fresh profile (agent). A file that exists but cannot be parsed must not read as + "agent holds": a torn write must never let the agent act on a human's screen. Taking over after a + request keeps the agent's reason so the human still sees WHY while they act.""" + from hermes_constants import hermes_home_key + + home = str(tmp_path) + assert lease.get(profile_key=home).holder == lease.AGENT + path = lease._path(home) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{ torn", encoding="utf-8") + assert lease.get(profile_key=home).holder == lease.HUMAN + lease.release(profile_key=home) # a successful write repairs it + assert lease.get(profile_key=home).holder == lease.AGENT + + # Valid JSON of the wrong shape is just as untrustworthy as torn JSON: never read it as "agent holds". + for wrong_shape in ("[]", "null", "5", "{}", '{"holder": "root"}'): + path.write_text(wrong_shape, encoding="utf-8") + assert lease.get(profile_key=home).holder == lease.HUMAN, wrong_shape + lease.release(profile_key=home) + assert lease.get(profile_key=home).holder == lease.AGENT + + lease.request_handoff("log in to the bank, 2FA on your phone", profile_key=home) + held = lease.acquire("desk-1", profile_key=home) + assert held.pending_handoff is None and held.reason == "log in to the bank, 2FA on your phone" + assert hermes_home_key(home) # sanity: the key derivation used by the bridge is available + + +def test_lease_works_without_fcntl(tmp_path): + """Windows and fcntl-less hosts: ``computer_use`` imports the lease (via handoff) on EVERY call, so a + module-level fcntl dependency turns every desktop action into ModuleNotFoundError there. The file + semantics must still work; only the cross-process lock degrades. Subprocess so the module cache is clean.""" + import os + import subprocess + import sys + + probe = ("import sys; sys.modules['fcntl'] = None; sys.path.insert(0, %r)\n" + "from tools.bot_desktop import lease\n" + "import tools.computer_use.handoff\n" + "assert lease.get().holder == lease.AGENT\n" + "assert lease.acquire('v1').holder == lease.HUMAN\n" + "assert lease.get().holder == lease.HUMAN\n" + "assert lease.release('v1').holder == lease.AGENT\n" + "print('OK')\n") % os.getcwd() + out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, encoding="utf-8", timeout=60, + stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": str(tmp_path)}) + assert out.stdout.strip() == "OK", out.stderr diff --git a/tests/tools/test_bot_desktop_rfb_filter_limits.py b/tests/tools/test_bot_desktop_rfb_filter_limits.py new file mode 100644 index 0000000000000..018336d9f6406 --- /dev/null +++ b/tests/tools/test_bot_desktop_rfb_filter_limits.py @@ -0,0 +1,24 @@ +"""A client-declared ClientCutText length is bounded at the header: the bridge must not buffer up to +2 GiB for a viewer that holds a ticket but no lease.""" + +import pytest + +from tools.bot_desktop.rfb_filter import _MAX_CUT_TEXT, RfbClientFilter + +_HANDSHAKE = b"RFB 003.008\n\x01\x01" + + +def clipboard_header(length): + return b"\x06\x00\x00\x00" + length.to_bytes(4, "big", signed=True) + + +@pytest.mark.parametrize("length", [_MAX_CUT_TEXT + 1, -_MAX_CUT_TEXT - 1, 2**31 - 1, -(2**31)]) +@pytest.mark.parametrize("holder", [False, True]) +def test_oversized_clipboard_is_rejected_at_header_without_waiting_for_payload(length, holder): + parser = RfbClientFilter(lambda: holder) + parser.feed(_HANDSHAKE) + header = clipboard_header(length) + for byte in header[:-1]: + assert parser.feed(bytes([byte])) == b"" + with pytest.raises(ValueError, match="clipboard"): + parser.feed(header[-1:]) diff --git a/tests/tools/test_bot_desktop_runtime.py b/tests/tools/test_bot_desktop_runtime.py new file mode 100644 index 0000000000000..fd89a14d6de8f --- /dev/null +++ b/tests/tools/test_bot_desktop_runtime.py @@ -0,0 +1,146 @@ +"""Bot Desktop runtime: thumbnail and display-number allocation invariants.""" + +from __future__ import annotations + +import contextlib +import sys +from pathlib import Path + +import pytest + +from tools.bot_desktop import runtime, thumbnail + + +@pytest.mark.parametrize("pm", sorted(runtime.PACKAGES)) +def test_every_required_binary_maps_to_an_installed_package(pm): + """Each binary the launcher execs must come from a package the distro list actually installs; dnf5 + refuses the whole transaction on one retired name, so the map is the contract, not the list.""" + mapping = runtime.BINARY_PACKAGES[pm] + assert set(mapping) == set(runtime.REQUIRED_BINARIES) + assert set(mapping.values()) <= set(runtime.PACKAGES[pm]) + assert not {"xorg-x11-server-utils", "xorg-x11-utils"} & set(runtime.PACKAGES["dnf"]), "retired on Fedora" + + +def test_no_running_screen_returns_none_without_grabbing(monkeypatch): + monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":99"}) + monkeypatch.setattr(runtime, "_launcher_pid", lambda: None) + monkeypatch.setitem(sys.modules, "PIL.ImageGrab", None) # an import would now fail loudly + assert thumbnail.thumbnail_data_url() is None + + +def test_recycled_pid_is_not_our_launcher(tmp_path, monkeypatch): + """launcher.pid names pid + create_time; a live pid born at another time is a stranger (recycled pid) + and must read as not running, or stop() would killpg an unrelated session. Legacy single-number + files and absurd digit strings are also not running.""" + import os + + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path) + pidfile = tmp_path / "launcher.pid" + pidfile.write_text(f"{os.getpid()} 12345.0", encoding="utf-8") # alive, wrong birth + assert runtime._launcher_pid() is None + pidfile.write_text(str(os.getpid()), encoding="utf-8") # pre-identity format + assert runtime._launcher_pid() is None + pidfile.write_text("9" * 40 + " 1.0", encoding="utf-8") + assert runtime._launcher_pid() is None + pidfile.write_text(f"{os.getpid()} {runtime._create_time(os.getpid())}", encoding="utf-8") + assert runtime._launcher_pid() == os.getpid() + + +def test_recorded_display_held_by_a_live_server_is_not_reused(tmp_path, monkeypatch): + """After profile A stops, B may take A's number; A restarting must pick another rather than + unlink B's socket and lock.""" + import os + + monkeypatch.setattr(runtime, "state_dir", lambda: tmp_path) + (tmp_path / "display").write_text("37", encoding="utf-8") + live = {37: os.getpid()} # :37 is owned by a running server (this very process stands in for it) + monkeypatch.setattr(runtime, "_display_in_use", lambda num: num in live) + monkeypatch.setattr(runtime, "_ALLOC_LOCK", tmp_path / "alloc.lock") + assert runtime._allocate_display() != 37 + live.clear() + assert runtime._allocate_display() == 37, "a free recorded number is reclaimed" + + + +_FAKE_LAUNCHER = """#!/usr/bin/env bash +# Stands in for launcher.sh + Xvnc: the X lock appears only after a delay (the TOCTOU window), then the +# env file + socket are published; stays alive until killed like the real supervisor. +: > "$HERMES_BD_XLOCK_DIR/spawned.$$" +sleep 0.4 +echo $$ > "$HERMES_BD_XLOCK_DIR/.X${HERMES_BD_DISPLAY_NUM}-lock" +: > "$HERMES_BD_SOCKET" +printf 'DISPLAY=:%s\\n' "$HERMES_BD_DISPLAY_NUM" > "$HERMES_BD_ENV_FILE" +sleep 30 +""" + +# One start() per process: state_dir() is HERMES_HOME-scoped and process-global, so two profiles need two +# interpreters — which is also how two gateway profiles race on a real host. +_DRIVER = """ +import json, os, sys +from pathlib import Path +sys.path.insert(0, {repo!r}) +from tools.bot_desktop import runtime +scratch = Path({scratch!r}) +runtime._LAUNCHER = scratch / "launcher.sh" +runtime._X_LOCK_DIR = scratch / "xlocks" +runtime._ALLOC_LOCK = scratch / "alloc.lock" +runtime.missing_binaries = lambda: [] +runtime.geometry = lambda: "800x600" +os.environ["HERMES_BD_XLOCK_DIR"] = str(scratch / "xlocks") +try: + st = runtime.start(wait_seconds=10) + print(json.dumps({{"display": st.display, "pid": st.pid}}), flush=True) +except Exception as exc: + print(json.dumps({{"error": str(exc)}}), flush=True) +sys.stdin.readline() # the test releases us once every driver has reported; we own the launcher, we stop it +runtime.stop() +""" + + +@pytest.fixture +def start_in_fresh_process(tmp_path): + import os + import subprocess + + (tmp_path / "launcher.sh").write_text(_FAKE_LAUNCHER, encoding="utf-8") + (tmp_path / "xlocks").mkdir() + repo = str(Path(__file__).resolve().parents[2]) + procs: list[subprocess.Popen] = [] + + def launch(home: Path) -> subprocess.Popen: + env = {**os.environ, "HERMES_HOME": str(home)} + proc = subprocess.Popen([sys.executable, "-c", _DRIVER.format(repo=repo, scratch=str(tmp_path))], + env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + procs.append(proc) + return proc + + yield launch + for proc in procs: + with contextlib.suppress(OSError): + proc.communicate("go\n", timeout=20) + proc.kill() + + +def _collect(procs): + import json + out = [json.loads(p.stdout.readline()) for p in procs] # every driver holds its launcher until released + assert all("error" not in o for o in out), out + return out + + +@pytest.mark.linux_only +def test_concurrent_cold_starts_of_two_profiles_get_distinct_displays(tmp_path, start_in_fresh_process): + """The allocation lock must outlive the pick: Xvnc writes /tmp/.X-lock well after start() chose n, so + a second profile starting in that window used to pick the same n (and its launcher's stale-lock cleanup + could then unlink the winner's socket).""" + out = _collect([start_in_fresh_process(tmp_path / "a"), start_in_fresh_process(tmp_path / "b")]) + assert len({o["display"] for o in out}) == 2, out + + +@pytest.mark.linux_only +def test_concurrent_starts_of_one_profile_spawn_one_launcher(tmp_path, start_in_fresh_process): + """Two start() calls for one profile spawn ONE launcher; the second used to spawn its own, overwrite + launcher.pid and orphan the first (both callers then reported the last-written pid).""" + out = _collect([start_in_fresh_process(tmp_path / "a"), start_in_fresh_process(tmp_path / "a")]) + assert len({o["pid"] for o in out}) == 1, out + assert len(list((tmp_path / "xlocks").glob("spawned.*"))) == 1 diff --git a/tests/tools/test_bot_desktop_thumbnail.py b/tests/tools/test_bot_desktop_thumbnail.py new file mode 100644 index 0000000000000..73f742c69afa2 --- /dev/null +++ b/tests/tools/test_bot_desktop_thumbnail.py @@ -0,0 +1,41 @@ +"""``thumbnail_data_url`` swaps the process-wide XAUTHORITY around the grab; two profiles grabbed on +worker threads at once must each see their own cookie file, not the other's.""" + +from __future__ import annotations + +import os +import threading + +from PIL import Image, ImageGrab + +from tools.bot_desktop import runtime, thumbnail + + +def test_concurrent_grabs_each_see_their_own_xauthority(monkeypatch): + envs = {"a": {"DISPLAY": ":91", "XAUTHORITY": "/tmp/xauth-a"}, + "b": {"DISPLAY": ":92", "XAUTHORITY": "/tmp/xauth-b"}} + local = threading.local() + monkeypatch.setattr(runtime, "published_env", lambda: envs[local.profile]) + monkeypatch.setattr(runtime, "_launcher_pid", lambda: 4242) + monkeypatch.delenv("XAUTHORITY", raising=False) + seen = {} + start = threading.Barrier(2) + + def fake_grab(xdisplay=None): + seen[xdisplay] = os.environ.get("XAUTHORITY") + threading.Event().wait(0.05) # hold the env long enough for the other thread to collide + return Image.new("RGB", (8, 8)) + monkeypatch.setattr(ImageGrab, "grab", fake_grab) + + def worker(profile): + local.profile = profile + start.wait() + thumbnail.thumbnail_data_url() + + threads = [threading.Thread(target=worker, args=(p,)) for p in envs] + for t in threads: + t.start() + for t in threads: + t.join(5) + assert seen == {":91": "/tmp/xauth-a", ":92": "/tmp/xauth-b"} + assert "XAUTHORITY" not in os.environ diff --git a/tests/tools/test_computer_use_backend_rebind.py b/tests/tools/test_computer_use_backend_rebind.py new file mode 100644 index 0000000000000..c2ca5b99071a1 --- /dev/null +++ b/tests/tools/test_computer_use_backend_rebind.py @@ -0,0 +1,13 @@ +"""A cached computer_use backend is bound to the display it spawned on; the identity helpers notice a change.""" + +from __future__ import annotations + +from tools.computer_use import cua_backend + + +def test_backend_display_identity_tracks_the_display_a_spawn_would_get(): + before = cua_backend.desktop_identity({"HOME": "/x"}) # no screen yet + after = cua_backend.desktop_identity({"HOME": "/x", "DISPLAY": ":37"}) # Bot Desktop came up + assert before == "" and after == ":37" + assert cua_backend.backend_display_stale(before, after) + assert not cua_backend.backend_display_stale(after, cua_backend.desktop_identity({"DISPLAY": ":37"})) diff --git a/tests/tools/test_computer_use_capture_fence.py b/tests/tools/test_computer_use_capture_fence.py new file mode 100644 index 0000000000000..65736e5ca12ef --- /dev/null +++ b/tests/tools/test_computer_use_capture_fence.py @@ -0,0 +1,56 @@ +"""A frame captured across a human takeover never leaves the process: the lease fence runs as soon as the +backend hands the frame back, before it is persisted to the media cache, spilled, or routed to aux vision.""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +from tools.bot_desktop import lease +from tools.computer_use.backend import ActionResult, CaptureResult + + +@pytest.fixture(autouse=True) +def _fresh_lease(): + lease._reset_for_tests() + yield + lease._reset_for_tests() + + +class _TakeoverBackend: + """Driver whose capture returns while a take-over / hand-back cycle happened underneath it.""" + _last_target = None + _last_app = None + + def capture(self, **kw): + lease.acquire("human") + lease.release("human") + return CaptureResult(mode="som", width=64, height=64, png_b64=base64.b64encode(b"SECRETPNG").decode(), elements=[], + app="Bank", window_title="login") + + def click(self, **kw): + return ActionResult(ok=True, action="click") + + +def _spy_sinks(monkeypatch, tool): + leaked: list = [] + monkeypatch.setattr(tool, "_persist_capture_image", lambda cap: leaked.append(("persist", cap))) + monkeypatch.setattr(tool, "_spill_elements_to_file", lambda cap: leaked.append(("spill", cap))) + monkeypatch.setattr(tool, "_should_route_through_aux_vision", lambda: leaked.append(("aux-decide",)) or True) + monkeypatch.setattr(tool, "_route_capture_through_aux_vision", lambda cap, summary, **kw: leaked.append(("aux", cap))) + return leaked + + +@pytest.mark.parametrize("args", [{"action": "capture"}, {"action": "click", "coordinate": [1, 1], "capture_after": True}]) +def test_frame_captured_across_a_takeover_is_dropped_before_any_sink(monkeypatch, args): + from tools.computer_use import tool + + monkeypatch.setattr(tool, "_get_backend", lambda session_id="": _TakeoverBackend()) + monkeypatch.setattr(tool, "_request_approval", lambda *a, **k: None) + leaked = _spy_sinks(monkeypatch, tool) + res = json.loads(tool.handle_computer_use(args)) + assert res["code"] == "human_has_control", res + assert leaked == [], f"the human's frame reached a sink: {leaked}" + assert "SECRETPNG" not in json.dumps(res) diff --git a/tests/tools/test_computer_use_handoff_wait.py b/tests/tools/test_computer_use_handoff_wait.py new file mode 100644 index 0000000000000..29811d8580540 --- /dev/null +++ b/tests/tools/test_computer_use_handoff_wait.py @@ -0,0 +1,38 @@ +"""wait_for_human answers an unanswered handoff early instead of blocking the whole timeout, and still +waits the full timeout once a human actually holds the screen.""" + +from __future__ import annotations + +import json +import threading +import time + +import pytest + +from tools.bot_desktop import lease +from tools.computer_use.handoff import handle_handoff + + +@pytest.fixture(autouse=True) +def _fresh_lease(): + lease._reset_for_tests() + yield + lease._reset_for_tests() + + +def test_wait_for_human_returns_no_takeover_when_nobody_answers_but_waits_out_a_real_takeover(): + handle_handoff("request_handoff", {"reason": "log in"}) + t0 = time.monotonic() + res = json.loads(handle_handoff("wait_for_human", {"seconds": 30, "grace": 0.2})) + assert res["code"] == "no_takeover" and res["state"]["pending_handoff"] == "log in" + assert time.monotonic() - t0 < 10, "an unanswered request must not run the full timeout" + + # A human takes over inside the grace window and hands back later: the wait outlives the grace. + def _take_then_release(): + time.sleep(0.1) + lease.acquire("viewer-1") + time.sleep(0.6) + lease.release("viewer-1") + threading.Thread(target=_take_then_release, daemon=True).start() + res = json.loads(handle_handoff("wait_for_human", {"seconds": 30, "grace": 0.3})) + assert res["ok"] and res["state"]["holder"] == lease.AGENT diff --git a/tests/tools/test_computer_use_native_platform.py b/tests/tools/test_computer_use_native_platform.py new file mode 100644 index 0000000000000..4a018eb39fb99 --- /dev/null +++ b/tests/tools/test_computer_use_native_platform.py @@ -0,0 +1,38 @@ +"""Exercise existing computer-use actions on the actual Windows/macOS CI hosts.""" + +import json +import sys + +import pytest + +from tools.computer_use import tool + + +@pytest.fixture +def backend(monkeypatch): + tool.reset_backend_for_tests() + monkeypatch.setenv("HERMES_COMPUTER_USE_BACKEND", "noop") + value = tool._get_backend() + yield value + tool.reset_backend_for_tests() + + +@pytest.mark.parametrize("host_platform", [ + pytest.param("win32", marks=pytest.mark.windows_only), + pytest.param("darwin", marks=pytest.mark.macos_only), +]) +@pytest.mark.parametrize("args, expected_call", [ + ({"action": "capture", "mode": "ax"}, "capture"), + ({"action": "click", "coordinate": [10, 10]}, "click"), + ({"action": "type", "text": "test input"}, "type"), + ({"action": "list_windows"}, "list_windows"), +]) +def test_native_computer_use_dispatches_to_backend(backend, args, expected_call, host_platform): + import tools.computer_use_tool # noqa: F401 - register the real tool handler + from tools.registry import registry + + assert sys.platform == host_platform + result = registry.dispatch("computer_use", args) + + assert "error" not in json.loads(result) + assert [name for name, _ in backend.calls] == [expected_call] diff --git a/tests/tui_gateway/test_display_methods.py b/tests/tui_gateway/test_display_methods.py new file mode 100644 index 0000000000000..4998946b47fa0 --- /dev/null +++ b/tests/tui_gateway/test_display_methods.py @@ -0,0 +1,125 @@ +"""display.install runs its worker inside the caller's profile scope; display.observe mints the viewer identity.""" + +from __future__ import annotations + +import hashlib +import json +import threading + +import pytest + +from hermes_cli.dashboard_auth import ws_tickets + + +def test_install_worker_keeps_the_requested_profile_scope(tmp_path, monkeypatch): + from hermes_constants import get_hermes_home + from tools.bot_desktop import install, runtime + import tui_gateway.server as server + + named = tmp_path / "profiles" / "named" + named.mkdir(parents=True) + monkeypatch.setattr(server, "_profile_home", lambda name: str(named) if name == "named" else None) + monkeypatch.setattr(runtime, "is_supported_host", lambda: True) + monkeypatch.setattr(runtime, "install_command", lambda: "sudo apt-get install -y x") + seen = {} + done = threading.Event() + + def fake_install(*, ask_password, on_line, timeout_seconds=900.0, claimed=False): + seen["home"] = str(get_hermes_home()) + done.set() + return 0 + + monkeypatch.setattr(install, "install_packages", fake_install) + monkeypatch.setattr(server, "_broadcast_global_event", lambda *a, **k: None) + resp = server.handle_request({"jsonrpc": "2.0", "id": 1, "method": "display.install", "params": {"profile": "named"}}) + assert resp["result"]["started"], resp + assert done.wait(5) + assert seen["home"] == str(named) + + +@pytest.fixture +def _fresh_lease(): + from tools.bot_desktop import lease + lease._reset_for_tests() + yield lease + lease._reset_for_tests() + + +def _call(server, method, params): + return server.handle_request({"jsonrpc": "2.0", "id": 7, "method": method, "params": params}) + + +def test_thumbnail_is_suppressed_while_a_human_holds_the_lease(monkeypatch, _fresh_lease): + """The Desktop polls thumbnails on a timer; while a human drives the screen that grab would ship + whatever they are typing to every connected client, so it must not touch the framebuffer at all.""" + import tui_gateway.server as server + from tools.bot_desktop import thumbnail + + grabs = [] + monkeypatch.setattr(thumbnail, "thumbnail_data_url", lambda: grabs.append(1) or "data:image/jpeg;base64,SECRET") + _fresh_lease.acquire("viewer-1") + result = _call(server, "display.thumbnail", {})["result"] + assert result["data_url"] is None and result["suppressed"] == "human_has_control" + assert grabs == [], "the framebuffer was grabbed while a human held the lease" + _fresh_lease.release("viewer-1") + assert _call(server, "display.thumbnail", {})["result"]["data_url"].endswith("SECRET") + + +def test_release_without_viewer_id_cannot_yank_another_viewers_lease(_fresh_lease): + """lease.release(None) skips the holder check, so a client that lost its viewer id (or a bare RPC) + must be refused unless it forces; a matching viewer id and force keep working.""" + import tui_gateway.server as server + + _fresh_lease.acquire("viewer-1") + refused = _call(server, "display.lease.release", {}) + assert refused["error"]["data"]["code"] == "viewer_mismatch" + assert _fresh_lease.get().holder == _fresh_lease.HUMAN + assert _call(server, "display.lease.release", {"viewer_id": "viewer-1"})["result"]["lease"]["holder"] == _fresh_lease.AGENT + _fresh_lease.acquire("viewer-2") + assert _call(server, "display.lease.release", {"force": True})["result"]["lease"]["holder"] == _fresh_lease.AGENT + +def _rpc(server, method, params): + return server.handle_request({"jsonrpc": "2.0", "id": 7, "method": method, "params": params}) + + +def test_observe_mints_the_viewer_id_and_status_never_discloses_the_holder(monkeypatch, tmp_path): + """A client cannot choose its viewer id (it would impersonate the holder and co-drive or release + their lease), and no snapshot or broadcast carries the raw holder id — only a hash the holder + itself can match.""" + from tools.bot_desktop import lease, runtime + import tui_gateway.server as server + + monkeypatch.setattr(runtime, "rfb_socket_path", lambda: tmp_path / "rfb.sock") + lease._reset_for_tests() + broadcasts = [] + monkeypatch.setattr(server, "_broadcast_global_event", lambda ev, payload=None: broadcasts.append((ev, payload))) + try: + observed = _rpc(server, "display.observe", {"viewer_id": "victim"})["result"] + assert observed["viewer_id"] != "victim" + assert observed["viewer_id"] and len(observed["viewer_id"]) >= 16 + assert ws_tickets.consume_ticket(observed["ticket"])["viewer_id"] == observed["viewer_id"] + holder = observed["viewer_id"] + + # Only the connection that minted an id may reuse it (a reconnecting pane keeps its lease). + class _Peer: + def write(self, obj): + return True + mine, other = _Peer(), _Peer() + with_mine = server.dispatch({"jsonrpc": "2.0", "id": 8, "method": "display.observe", "params": {}}, mine)["result"] + again = server.dispatch({"jsonrpc": "2.0", "id": 9, "method": "display.observe", + "params": {"viewer_id": with_mine["viewer_id"]}}, mine)["result"] + assert again["viewer_id"] == with_mine["viewer_id"] + stolen = server.dispatch({"jsonrpc": "2.0", "id": 10, "method": "display.observe", + "params": {"viewer_id": with_mine["viewer_id"]}}, other)["result"] + assert stolen["viewer_id"] != with_mine["viewer_id"] + + _rpc(server, "display.status", {}) # installs the broadcast listener + lease.acquire(holder) + status = _rpc(server, "display.status", {})["result"] + assert status["lease"]["holder"] == lease.HUMAN + assert holder not in json.dumps(status) + assert status["lease"]["viewer_hash"] == hashlib.sha256(holder.encode()).hexdigest()[:12] + lease_events = [p for ev, p in broadcasts if ev == "display.lease"] + assert lease_events and all(holder not in json.dumps(p) for p in lease_events) + finally: + lease._reset_for_tests() diff --git a/tests/tui_gateway/test_display_watch.py b/tests/tui_gateway/test_display_watch.py new file mode 100644 index 0000000000000..b9248724c9d2e --- /dev/null +++ b/tests/tui_gateway/test_display_watch.py @@ -0,0 +1,80 @@ +"""Lease transitions made by ANOTHER process reach `hermes serve` clients as ``display.lease``. + +The agent lives in whatever process hosts it (messaging gateway, ``hermes chat``, a cron worker); +``lease.on_change`` is in-process only, so the serve backend must notice the file move itself. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] + + +def _other_process(home: Path, stmt: str) -> None: + env = {**os.environ, "HERMES_HOME": str(home)} + subprocess.run( # noqa: S603 + [sys.executable, "-c", f"import sys; sys.path.insert(0, {str(REPO)!r}); " + f"from tools.bot_desktop import lease; {stmt}"], + cwd=str(REPO), env=env, check=True, timeout=60, stdin=subprocess.DEVNULL) + + +def _wait_for(pred, timeout: float = 3.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(0.05) + return pred() + + +def _watching(server, home: Path, monkeypatch) -> list: + from hermes_constants import hermes_home_key + events: list = [] + monkeypatch.setattr(server, "_broadcast_global_event", lambda ev, payload=None: events.append((ev, payload))) + monkeypatch.setattr(server, "_hermes_home", str(home)) + server._ensure_lease_watcher() + assert _wait_for(lambda: hermes_home_key(home) in server._lease_epochs), "watcher never seeded the home" + return events + + +def _lease_events(events, **want): + return [p for ev, p in events if ev == "display.lease" and all(p["lease"].get(k) == v for k, v in want.items())] + + +def test_handoff_requested_in_another_process_is_broadcast(tmp_path, monkeypatch): + import tui_gateway.server as server + from hermes_constants import hermes_home_key + home = tmp_path / "home" + home.mkdir() + events = _watching(server, home, monkeypatch) + + _other_process(home, 'lease.request_handoff("probe")') + + assert _wait_for(lambda: _lease_events(events, pending_handoff="probe")), events + (payload,) = _lease_events(events, pending_handoff="probe") + assert payload["profile_key"] == hermes_home_key(home) + + +def test_release_in_another_process_is_broadcast_and_local_transition_not_duplicated(tmp_path, monkeypatch): + import tui_gateway.server as server + from tools.bot_desktop import lease + home = tmp_path / "home" + home.mkdir() + events = _watching(server, home, monkeypatch) + server._install_lease_listener() # what display.status does: in-process transitions broadcast here + + lease.acquire("viewer-1", profile_key=str(home)) + assert _wait_for(lambda: _lease_events(events, holder="human")) + before = len(events) + server._poll_lease_files() # the file moved too, but the watcher saw that epoch locally: no re-broadcast + assert len(events) == before + + _other_process(home, 'lease.release("viewer-1")') + + assert _wait_for(lambda: _lease_events(events, holder="agent")), events + assert len(_lease_events(events, holder="agent")) == 1 diff --git a/tools/bot_desktop/__init__.py b/tools/bot_desktop/__init__.py new file mode 100644 index 0000000000000..2ffa400fd64e5 --- /dev/null +++ b/tools/bot_desktop/__init__.py @@ -0,0 +1,5 @@ +"""Bot Desktop: per-profile headless Xfce desktop over RFB, viewed from Hermes Desktop. + +``runtime`` owns the Xvnc/Xfce process and the published env; ``lease`` owns who may drive the +screen (agent vs. human); ``rfb_filter`` is the byte-level input gate the WebSocket bridge applies. +""" diff --git a/tools/bot_desktop/browser.py b/tools/bot_desktop/browser.py new file mode 100644 index 0000000000000..08d7489ee3d81 --- /dev/null +++ b/tools/bot_desktop/browser.py @@ -0,0 +1,121 @@ +"""The bot's browser on its Bot Desktop: one executable, one persistent user-data-dir per profile. + +The agent drives Chromium through agent-browser; a human who takes over clicks the dock's Browser +icon. Both must be THE SAME browser — same binary, same ``--user-data-dir`` — or the human logs in +to a jar the bot never sees. Chromium's singleton makes a second launch on the same user-data-dir +open a window in the running instance, which is exactly the hand-over we want — in ONE direction. When the +human's dock instance is already up, agent-browser's own launch is forwarded to it and dies without a +DevTools endpoint, so the dock exposes a debugging port and the agent ATTACHES to it (see +:func:`running_instance_cdp_port`) instead of launching. +""" + +from __future__ import annotations + +import glob +import os +import shutil +import socket +from pathlib import Path +from typing import Optional, Tuple + +from tools.bot_desktop import runtime + +_SYSTEM_BROWSERS = ("google-chrome", "google-chrome-stable", "chromium", "chromium-browser") + + +def profile_dir() -> Path: + """User-data-dir the bot's browser uses on this profile's screen (``AGENT_BROWSER_PROFILE`` wins).""" + override = os.environ.get("AGENT_BROWSER_PROFILE", "").strip() + if override and os.path.isabs(override): + return Path(override) + return runtime.state_dir() / "browser-profile" + + +def executable() -> Optional[str]: + """The Chromium agent-browser launches: an explicit ``AGENT_BROWSER_EXECUTABLE_PATH``, else the newest + Playwright Chromium it bundles, else a system Chrome/Chromium. ``None`` when there is none.""" + explicit = os.environ.get("AGENT_BROWSER_EXECUTABLE_PATH", "").strip() + if explicit and os.access(explicit, os.X_OK): + return explicit + from tools.browser_tool_install import _chromium_search_roots + candidates = sorted( + (p for root in _chromium_search_roots() for p in glob.glob(os.path.join(root, "chromium-*", "chrome-linux*", "chrome"))), + key=os.path.getmtime, reverse=True) + for exe in candidates: + if os.access(exe, os.X_OK): + return exe + return next((shutil.which(name) for name in _SYSTEM_BROWSERS if shutil.which(name)), None) + + +def dock_launch() -> Optional[Tuple[str, str]]: + """``(executable, user_data_dir)`` for the dock's Browser icon, or ``None`` when no Chromium exists.""" + exe = executable() + return (exe, str(profile_dir())) if exe else None + + +def dock_command(exe: str, user_data_dir: str) -> str: + """Shell line the dock's Browser icon runs. ``--remote-debugging-port=0`` makes a human-started + instance attachable (Chromium writes the chosen port to ``/DevToolsActivePort``); + first-run / default-browser dialogs would sit between the human and the bot's tabs.""" + return f"{exe} --user-data-dir={user_data_dir} --remote-debugging-port=0 --no-first-run --no-default-browser-check" + + +def running_instance_cdp_port(user_data_dir: str, *, exclude_session: Optional[str] = None) -> Optional[int]: + """DevTools port of a Chromium currently running on ``user_data_dir``, or ``None``. + + Both files outlive a crashed or closed Chromium: ``SingletonLock`` is a symlink to ``host-pid`` and + ``DevToolsActivePort`` keeps the last port, so the pid must be alive AND the port must accept a + connection before it is trusted. An instance agent-browser launched for ``exclude_session`` itself is + reported as ``None``: its daemon already owns that browser, and handing it ``--cdp`` would make it + close the browser as a config change and then attach to the port that just died with it. + """ + try: + with open(os.path.join(user_data_dir, "DevToolsActivePort"), encoding="utf-8") as fh: + port_line = fh.readline().strip() + target = os.readlink(os.path.join(user_data_dir, "SingletonLock")) + except OSError: + return None + _host, _, pid_text = target.rpartition("-") + if not (port_line.isdigit() and pid_text.isdigit()) or not _pid_alive(int(pid_text)): + return None + if exclude_session and _launched_by_session(int(pid_text)) == exclude_session: + return None + port = int(port_line) + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + pass + except OSError: + return None + return port + + +def _launched_by_session(chromium_pid: int) -> Optional[str]: + """``AGENT_BROWSER_SESSION`` of the agent-browser daemon that spawned ``chromium_pid``, or ``None`` + for a human-started (dock) instance. Chromium itself gets a scrubbed environment, so the daemon's + ``/proc//environ`` is the marker (Linux-only, same user).""" + try: + with open(f"/proc/{chromium_pid}/status", encoding="utf-8") as fh: + ppid = next((int(line.split()[1]) for line in fh if line.startswith("PPid:")), 0) + with open(f"/proc/{ppid}/environ", "rb") as fh: + raw = fh.read() + except (OSError, ValueError): + return None + for item in raw.split(b"\0"): + key, sep, value = item.partition(b"=") + if sep and key == b"AGENT_BROWSER_SESSION": + return value.decode("utf-8", "replace") or None + return None + + +def _pid_alive(pid: int) -> bool: + import psutil + return psutil.pid_exists(pid) + + +def env_for_agent(env: dict) -> dict: + """Pin agent-browser to the screen's browser identity unless the user pinned their own.""" + env.setdefault("AGENT_BROWSER_PROFILE", str(profile_dir())) + exe = executable() + if exe: + env.setdefault("AGENT_BROWSER_EXECUTABLE_PATH", exe) + return env diff --git a/tools/bot_desktop/install.py b/tools/bot_desktop/install.py new file mode 100644 index 0000000000000..ee44b7e06b5c9 --- /dev/null +++ b/tools/bot_desktop/install.py @@ -0,0 +1,125 @@ +"""Install the Bot Desktop packages on the gateway host from a Desktop client. + +The install runs the distro command from ``runtime.install_command()`` (apt/dnf/pacman) as a child +process on THIS host. Privilege comes from the same masked ``sudo.request`` card the terminal tool +raises: ``sudo -n true`` is probed first (NOPASSWD / cached timestamp hosts never see a prompt); when +a password is needed the caller-supplied ``ask_password`` blocks on the card and the value is written +to sudo's stdin (``-S``) exactly once, never logged, never placed on the command line. Output lines +stream through ``on_line`` so the pane can show apt's progress; the return value is the exit code. + +One install per profile at a time; a second request while one runs is refused. +""" + +from __future__ import annotations + +import contextlib +import logging +import os +import shlex +import signal +import subprocess +import threading +from typing import Callable, Optional + +from hermes_constants import hermes_home_key +from tools.bot_desktop import runtime + +logger = logging.getLogger(__name__) + +_install_lock = threading.Lock() +_running: set[str] = set() + + +class InstallBusy(RuntimeError): + pass + + +def assert_not_running() -> None: + with _install_lock: + if hermes_home_key() in _running: + raise InstallBusy("an install is already running for this profile") + + +def claim() -> str: + """Atomically take this profile's install slot; raises :class:`InstallBusy` when taken. A caller that + claims before handing off to a worker passes ``claimed=True`` to :func:`install_packages`, which then + owns releasing it — a check-then-spawn pair (``assert_not_running`` + later claim on the worker) lets + two Install clicks both pass the check.""" + key = hermes_home_key() + with _install_lock: + if key in _running: + raise InstallBusy("an install is already running for this profile") + _running.add(key) + return key + + +def release(key: str) -> None: + with _install_lock: + _running.discard(key) + + +def install_packages(*, ask_password: Callable[[], str], on_line: Callable[[str], None], + timeout_seconds: float = 900.0, claimed: bool = False) -> int: + """Run the package install; returns the process exit code (0 = success, ``-1`` = cancelled). + ``claimed=True``: the caller already holds the slot via :func:`claim`; it is released here either way.""" + key = hermes_home_key() if claimed else None + try: + if not runtime.is_supported_host(): + raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") + cmd = runtime.install_command() + if cmd is None: + raise RuntimeError("no supported package manager (apt-get, dnf, pacman) found on this host") + if key is None: + key = claim() + return _run(cmd, ask_password=ask_password, on_line=on_line, timeout_seconds=timeout_seconds) + finally: + if key is not None: + release(key) + + +def _sudo_nopasswd() -> bool: + try: + return subprocess.run(["sudo", "-n", "true"], capture_output=True, timeout=3, + stdin=subprocess.DEVNULL).returncode == 0 + except Exception: + return False + + +def _run(cmd: str, *, ask_password: Callable[[], str], on_line: Callable[[str], None], + timeout_seconds: float) -> int: + argv = shlex.split(cmd) + assert argv[0] == "sudo", cmd + stdin_payload: Optional[str] = None + if not _sudo_nopasswd(): + password = ask_password() or "" + if not password: + on_line("install cancelled: no sudo password provided") + return -1 + # -S: read the password from stdin; -p '': no prompt text mixed into the streamed output. + argv = ["sudo", "-S", "-p", "", *argv[1:]] + stdin_payload = password + "\n" + on_line(f"$ {cmd}") + env = {"DEBIAN_FRONTEND": "noninteractive", "LC_ALL": "C.UTF-8"} + proc = subprocess.Popen( # windows-footgun: ok — Linux-only (is_supported_host) + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env={**os.environ, **env}, text=True, encoding="utf-8", errors="replace", start_new_session=True) + try: + if stdin_payload is not None: + proc.stdin.write(stdin_payload) # type: ignore[union-attr] + proc.stdin.close() # type: ignore[union-attr] + except OSError: + pass + # The package manager runs in its own session (start_new_session); killing only sudo would leave apt/dnf + # running as root with the dpkg lock while the slot is released, so the whole group goes. + def _kill_group() -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) # windows-footgun: ok — Linux-only (is_supported_host) + + timer = threading.Timer(timeout_seconds, _kill_group) + timer.start() + try: + for line in proc.stdout: # type: ignore[union-attr] + on_line(line.rstrip("\n")) + return proc.wait() + finally: + timer.cancel() diff --git a/tools/bot_desktop/launcher.sh b/tools/bot_desktop/launcher.sh new file mode 100755 index 0000000000000..9eb106b1ed710 --- /dev/null +++ b/tools/bot_desktop/launcher.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# Hermes Bot Desktop — one headless Xfce desktop per Hermes profile, served over RFB. +# +# Spawned by tools/bot_desktop/runtime.py with HERMES_BD_* variables set. Runs TigerVNC's Xvnc +# (X server + RFB server in one process; damage-driven, resizable via SetDesktopSize) listening on a +# 0600 Unix socket only, then a minimal Xfce started component-wise under a private dbus session. +# +# Why not startxfce4 / xfce4-session: xfce4-session expects a logind session scope; outside one it +# spawns polkit agents that pop empty dialogs and light-locker/xfce4-screensaver lock the desktop for a +# user who has no password. Starting xfsettingsd -> xfwm4 -> xfdesktop -> xfce4-panel directly, with +# the screensaver/locker/power-manager autostarts masked, is the shape every headless-VNC recipe +# converges on (TigerVNC #1096/#581, OpenOnDemand's apptainer desktop, the Arch wiki). +# +# Why not the xfce4 metapackage: it drags in the screensaver, power manager and polkit agent this +# script exists to keep out. +set -euo pipefail + +: "${HERMES_BD_PROFILE:?}" # profile name (display name in the VNC title) +: "${HERMES_BD_DISPLAY_NUM:?}" # allocated by runtime.py +: "${HERMES_BD_SOCKET:?}" # RFB unix socket path +: "${HERMES_BD_XAUTH:?}" # Xauthority path +: "${HERMES_BD_ENV_FILE:?}" # where to publish DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS +: "${HERMES_BD_CONFIG_HOME:?}" # per-profile XDG_CONFIG_HOME (xfconf lives here) +GEOM="${HERMES_BD_GEOMETRY:-1440x900}" +DEPTH=24 + +export XDG_CONFIG_HOME="$HERMES_BD_CONFIG_HOME" +export XDG_CACHE_HOME="${HERMES_BD_CACHE_HOME:-$HERMES_BD_CONFIG_HOME/.cache}" +export XDG_DATA_HOME="${HERMES_BD_DATA_HOME:-$HERMES_BD_CONFIG_HOME/.local-share}" +export XDG_SESSION_TYPE=x11 XDG_CURRENT_DESKTOP=XFCE +export GDK_BACKEND=x11 QT_QPA_PLATFORM=xcb NO_AT_BRIDGE=1 GTK_A11Y=none +export LANG="${LANG:-C.UTF-8}" +# Inheriting a login session's bus/session manager yields "Another session manager is already +# running" / "Unable to contact settings server". +unset SESSION_MANAGER DBUS_SESSION_BUS_ADDRESS DISPLAY XAUTHORITY WAYLAND_DISPLAY + +mkdir -p "$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" "$XDG_CONFIG_HOME/autostart" \ + "$XDG_CACHE_HOME" "$XDG_DATA_HOME" "$(dirname "$HERMES_BD_SOCKET")" + +export DISPLAY=":$HERMES_BD_DISPLAY_NUM" +export XAUTHORITY="$HERMES_BD_XAUTH" + +# Stale lock files from a crashed server block restart; a lock whose pid is alive belongs to a +# running server (another profile may have taken this number) and is never touched — Xvnc then +# fails to start on it and runtime.py reports that instead of us disrupting the other desktop. +rm -f "$HERMES_BD_SOCKET" +xlock="/tmp/.X${HERMES_BD_DISPLAY_NUM}-lock" +if [[ -e "$xlock" ]] && ! kill -0 "$(tr -d ' ' < "$xlock" 2>/dev/null)" 2>/dev/null; then + rm -f "$xlock" "/tmp/.X11-unix/X${HERMES_BD_DISPLAY_NUM}" +fi +: > "$XAUTHORITY"; chmod 600 "$XAUTHORITY" +xauth -q -f "$XAUTHORITY" add "$DISPLAY" MIT-MAGIC-COOKIE-1 "$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" + +# ---- look: dark theme from whatever the host ships (first match wins), Hermes wallpaper ---- +pick_theme() { local d t; for t in "$@"; do for d in /usr/share/themes "$HOME/.themes"; do [[ -d "$d/$t" ]] && { echo "$t"; return; }; done; done; echo "$1"; } +pick_icons() { local d t; for t in "$@"; do for d in /usr/share/icons "$HOME/.icons"; do [[ -d "$d/$t" ]] && { echo "$t"; return; }; done; done; echo "$1"; } +GTK_THEME_NAME=$(pick_theme Adwaita-dark Breeze-Dark Greybird-dark Arc-Dark Adwaita) +WM_THEME_NAME=$(pick_theme Default-hdpi Default) # xfwm4 window themes ship with xfwm4 itself +ICON_THEME_NAME=$(pick_icons Papirus-Dark breeze-dark Adwaita hicolor) +: "${HERMES_BD_WALLPAPER:="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/wallpaper.png"}" + +# ---- pre-seed xfconf BEFORE xfconfd starts (it caches; edits after start are overwritten) ---- +X="$XDG_CONFIG_HOME/xfce4/xfconf/xfce-perchannel-xml" +[[ -e "$X/xfwm4.xml" ]] || cat > "$X/xfwm4.xml" <<'EOF' + + + + + + + + + + +EOF +sed -i "s|HERMES_BD_WM_THEME|$WM_THEME_NAME|" "$X/xfwm4.xml" +[[ -e "$X/xfce4-screensaver.xml" ]] || cat > "$X/xfce4-screensaver.xml" <<'EOF' + + + + + +EOF +[[ -e "$X/xsettings.xml" ]] || cat > "$X/xsettings.xml" <<'EOF' + + + + + + + + + + + + + + + + + +EOF +sed -i "s|HERMES_BD_GTK_THEME|$GTK_THEME_NAME|; s|HERMES_BD_ICON_THEME|$ICON_THEME_NAME|" "$X/xsettings.xml" +[[ -e "$X/xfce4-desktop.xml" ]] || cat > "$X/xfce4-desktop.xml" <<'EOF' + + + + + + + + + + + + + + + + + + + + + +EOF +sed -i "s|HERMES_BD_WALLPAPER_PLACEHOLDER|$HERMES_BD_WALLPAPER|" "$X/xfce4-desktop.xml" +# Own panel layout (a layout on disk also suppresses the first-run "Welcome to the panel" dialog): +# top bar = menu · tasks · tray · clock; bottom dock = only launchers whose program exists on this +# host, the browser pinned to the one the bot drives so a human lands in the bot's own browser profile. +if [[ ! -e "$X/xfce4-panel.xml" ]]; then + L="$XDG_CONFIG_HOME/xfce4/panel"; mkdir -p "$L" + dock_ids=(); n=20 + add_launcher() { # name icon exec — skipped when the executable is missing + local exe; exe=${3%% *} + command -v "$exe" >/dev/null 2>&1 || return 0 + n=$((n+1)); mkdir -p "$L/launcher-$n" + printf '[Desktop Entry]\nVersion=1.0\nType=Application\nName=%s\nIcon=%s\nExec=%s\nTerminal=false\nStartupNotify=false\n' \ + "$1" "$2" "$3" > "$L/launcher-$n/hermes.desktop" + dock_ids+=("$n") + } + add_launcher "Terminal" utilities-terminal "xfce4-terminal" + # The bot's browser: runtime.py resolves the executable agent-browser drives plus the profile's + # persistent user-data-dir, so a human taking over lands in the bot's own cookie jar. + [[ -n "${HERMES_BD_BROWSER_EXEC:-}" ]] && add_launcher "Browser" internet-web-browser "$HERMES_BD_BROWSER_EXEC" + add_launcher "Files" system-file-manager "thunar" + add_launcher "Text Editor" accessories-text-editor "mousepad" + dock_plugins=""; dock_items="" + for id in "${dock_ids[@]}"; do + dock_plugins+="" + dock_items+="" + done + cat > "$X/xfce4-panel.xml" < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${dock_plugins} + + + + + + + + + + + + + + + + + + + ${dock_items} + + +PANEL +fi +# Mask system autostarts that want logind/polkit/keyring/at-spi. +for a in xfce4-screensaver light-locker xfce4-power-manager xfce-polkit \ + polkit-gnome-authentication-agent-1 lxpolkit xfce4-notifyd blueman at-spi-dbus-bus \ + gnome-keyring-pkcs11 gnome-keyring-secrets gnome-keyring-ssh xdg-user-dirs; do + [[ -e "$XDG_CONFIG_HOME/autostart/$a.desktop" ]] || \ + printf '[Desktop Entry]\nType=Application\nName=%s\nHidden=true\n' "$a" > "$XDG_CONFIG_HOME/autostart/$a.desktop" +done +# Tests seed the config tree on a fake PATH and stop here (no X server needed). +[[ -n "${HERMES_BD_SEED_ONLY:-}" ]] && exit 0 + +# ---- X server + RFB (TigerVNC Xvnc), Unix socket only ---- +# SecurityTypes None is safe ONLY because -rfbport -1 disables TCP and the 0600 socket is reachable +# only by processes running as this user (the gateway's WebSocket bridge does the real authentication; +# same-UID processes, the bot's own terminal tool included, are inside that boundary by design). +# -SendCutText=0: watchers must never receive the holder's clipboard; -AcceptCutText stays on so +# paste INTO the screen keeps working. +Xvnc "$DISPLAY" -geometry "$GEOM" -depth "$DEPTH" -dpi 96 \ + -rfbport -1 -rfbunixpath "$HERMES_BD_SOCKET" -rfbunixmode 0600 \ + -SecurityTypes None -AlwaysShared -AcceptSetDesktopSize -FrameRate 30 -SendCutText=0 \ + -desktop "hermes:$HERMES_BD_PROFILE" -auth "$XAUTHORITY" -nolisten tcp \ + -Log '*:stderr:30' & +XVNC_PID=$! +trap 'kill "$XVNC_PID" 2>/dev/null || true' EXIT +for _ in $(seq 1 100); do + xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 && break + kill -0 "$XVNC_PID" 2>/dev/null || { echo "Xvnc exited during startup" >&2; exit 1; } + sleep 0.1 +done +xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 || { echo "Xvnc did not become ready" >&2; exit 1; } + +setxkbmap -display "$DISPLAY" us 2>/dev/null || true # RFB keysyms + xdotool assume a known layout +xsetroot -display "$DISPLAY" -solid '#1c1f29' 2>/dev/null || true +xset -display "$DISPLAY" s off -dpms s noblank 2>/dev/null || true + +# ---- private session bus + Xfce components (no xfce4-session) ---- +# dbus-run-session scopes the bus to this subshell: no leaked dbus-daemons on restart. The env file +# is written from INSIDE the bus so DBUS_SESSION_BUS_ADDRESS is the real one; runtime.py and every +# cua-driver / browser spawn for this profile source it. +# Not exec'd: this script stays the supervisor so the EXIT trap above still reaps Xvnc when the Xfce +# session dies on its own (exec would replace the trap's owner and orphan the X server). +dbus-run-session -- bash -c ' + set -e + umask 077 + printf "DISPLAY=%s\nXAUTHORITY=%s\nDBUS_SESSION_BUS_ADDRESS=%s\nXDG_CONFIG_HOME=%s\nXDG_CACHE_HOME=%s\nXDG_DATA_HOME=%s\n" \ + "$DISPLAY" "$XAUTHORITY" "$DBUS_SESSION_BUS_ADDRESS" "$XDG_CONFIG_HOME" "$XDG_CACHE_HOME" "$XDG_DATA_HOME" \ + > "$HERMES_BD_ENV_FILE.tmp" && mv -f "$HERMES_BD_ENV_FILE.tmp" "$HERMES_BD_ENV_FILE" + xfsettingsd --sm-client-disable --daemon 2>/dev/null || true + xfwm4 --compositor=off --sm-client-disable & + for _ in $(seq 1 50); do xprop -root _NET_SUPPORTING_WM_CHECK >/dev/null 2>&1 && break; sleep 0.1; done + xfdesktop --sm-client-disable --disable-wm-check & + exec xfce4-panel --sm-client-disable --disable-wm-check +' && rc=0 || rc=$? +kill "$XVNC_PID" 2>/dev/null || true +exit "$rc" diff --git a/tools/bot_desktop/lease.py b/tools/bot_desktop/lease.py new file mode 100644 index 0000000000000..33b12f2009706 --- /dev/null +++ b/tools/bot_desktop/lease.py @@ -0,0 +1,243 @@ +"""Who may drive a profile's Bot Desktop screen: the agent (default) or exactly one human viewer. + +The lease is the single truth shared by the RFB bridge (drops human input from non-holders), the +``computer_use`` tool (refuses to act while a human holds control — the person may be typing a +credential, so even screenshots are refused; fail closed rather than trusting the agent to pause +itself) and the Desktop UI (Watch / Take over / Hand back). + +Authority lives ON DISK, ``/bot-desktop/lease.json`` under an fcntl lock, because the +processes that must agree do not share memory: ``hermes serve`` (viewer bridge), the messaging +gateway, a CLI turn and isolated workers all drive the same display. Every read goes to the file; +the in-process Condition only wakes local waiters early. ``epoch`` increments on every transition so +an action admitted under one lease can tell that control changed underneath it. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Callable, Dict, List, Optional + +from hermes_constants import get_hermes_home, hermes_home_key + +try: + import fcntl +except ImportError: # Windows/macOS without fcntl: computer_use imports this module on every call, and no + fcntl = None # multi-process Bot Desktop exists there, so the cross-process lock degrades to a no-op. + +AGENT = "agent" +HUMAN = "human" +_POLL_SECONDS = 0.25 + + +class HumanHasControl(RuntimeError): + """Raised by screen-driving tools while a human holds the lease.""" + + +@dataclass +class Lease: + holder: str = AGENT + viewer_id: Optional[str] = None + since: float = field(default_factory=time.time) + reason: str = "" + pending_handoff: Optional[str] = None # agent's reason for asking, until the human takes over + epoch: int = 0 + + def as_dict(self) -> Dict[str, object]: + return asdict(self) + + +_lock = threading.Condition() +_listeners: List[Callable[[str, Lease], None]] = [] + + +def _path(profile_key: Optional[str]) -> Path: + """``profile_key`` is the HERMES_HOME path of the profile whose lease is meant (the RFB bridge + serves several profiles from one process); ``None`` means the current profile.""" + home = Path(profile_key) if profile_key else get_hermes_home() + return home / "bot-desktop" / "lease.json" + + +def _read(path: Path) -> Lease: + """No file = fresh profile, agent holds. A file that exists but cannot be parsed is a torn write + or tampering: fail CLOSED (human holds) — an unreadable lease must never let the agent act on a + screen a human may be using; the next successful write repairs it.""" + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + return Lease() + except OSError: + return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file unreadable") + try: + data = json.loads(raw) + except ValueError: + data = None + if not isinstance(data, dict) or data.get("holder") not in (AGENT, HUMAN): + return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file corrupt") + try: + return Lease(**{k: v for k, v in data.items() if k in Lease.__dataclass_fields__}) + except TypeError: + return Lease(holder=HUMAN, viewer_id="unreadable-lease", reason="lease file corrupt") + + +def _write(path: Path, lease: Lease) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(lease.as_dict()), encoding="utf-8") + os.replace(tmp, path) + + +class _locked: + """Cross-process critical section over the lease file (fcntl on a sibling lock file).""" + + def __init__(self, path: Path): + self._lockfile = path.with_suffix(".lock") + self._fh = None + + def __enter__(self): + if fcntl is None: + return self + self._lockfile.parent.mkdir(parents=True, exist_ok=True) + self._fh = open(self._lockfile, "a+", encoding="utf-8") # noqa: SIM115 — closed in __exit__ + fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX) + return self + + def __exit__(self, *exc): + if self._fh is None: + return + fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN) + self._fh.close() + + +def get(profile_key: Optional[str] = None) -> Lease: + return _read(_path(profile_key)) + + +def on_change(listener: Callable[[str, Lease], None]) -> Callable[[], None]: + """Subscribe to lease transitions made IN THIS PROCESS (the gateway broadcasts them to Desktop + clients). Transitions made by another process are observed by reading, not by callback.""" + with _lock: + _listeners.append(listener) + + def _off() -> None: + with _lock: + if listener in _listeners: + _listeners.remove(listener) + return _off + + +def _notify(key: str, lease: Lease) -> None: + for cb in list(_listeners): + try: + cb(key, lease) + except Exception: # a broken subscriber must not wedge the handoff + pass + + +def _transition(profile_key: Optional[str], mutate: Callable[[Lease], bool]) -> Lease: + key, path = hermes_home_key(profile_key) if profile_key else hermes_home_key(), _path(profile_key) + with _locked(path): + lease = _read(path) + if not mutate(lease): + return lease + lease.epoch += 1 + _write(path, lease) + with _lock: + _lock.notify_all() + _notify(key, lease) + return lease + + +def acquire(viewer_id: str, *, profile_key: Optional[str] = None, reason: str = "") -> Lease: + """Human ``viewer_id`` takes control. Last writer wins: a second viewer evicts the first, and the + RFB bridge closes the evicted socket so its UI drops to view-only.""" + def _m(lease: Lease) -> bool: + # The agent's ask ("please log in to X") stays as the takeover reason: the human needs it + # on screen WHILE they act, not only before they clicked Take over. + lease.holder, lease.viewer_id, lease.since = HUMAN, viewer_id, time.time() + lease.reason = reason or lease.pending_handoff or "" + lease.pending_handoff = None + return True + return _transition(profile_key, _m) + + +def release(viewer_id: Optional[str] = None, *, profile_key: Optional[str] = None) -> Lease: + """Return control to the agent. With ``viewer_id`` only that holder may release (a stale viewer + closing its window must not yank control from the one who took over after it).""" + def _m(lease: Lease) -> bool: + if viewer_id is not None and lease.holder == HUMAN and lease.viewer_id != viewer_id: + return False + lease.holder, lease.viewer_id, lease.since, lease.reason = AGENT, None, time.time(), "" + lease.pending_handoff = None # "hand back" answers an open request even if nobody formally took over + return True + return _transition(profile_key, _m) + + +def request_handoff(reason: str, *, profile_key: Optional[str] = None) -> Lease: + """Agent asks a human to take over (login, 2FA, CAPTCHA, payment). Recorded so the UI can show + why and the bridge can page the user; control itself still flips only on ``acquire``.""" + def _m(lease: Lease) -> bool: + lease.pending_handoff = reason + return True + return _transition(profile_key, _m) + + +def wait_for_release(*, timeout: float, profile_key: Optional[str] = None) -> bool: + """Block until the agent holds the lease (and no handoff is pending) or ``timeout`` elapses. + True when control is back with the agent. Polls the file so a release made by another process + is seen; the local Condition just shortens the wait for same-process transitions.""" + return _wait_until(lambda lease: lease.holder == AGENT and lease.pending_handoff is None, timeout, profile_key) + + +def wait_for_takeover_or_release(*, timeout: float, profile_key: Optional[str] = None) -> bool: + """False when, after ``timeout``, the agent still holds with a handoff pending: nobody answered.""" + return _wait_until(lambda lease: lease.holder == HUMAN or lease.pending_handoff is None, timeout, profile_key) + + +def _wait_until(done: Callable[[Lease], bool], timeout: float, profile_key: Optional[str]) -> bool: + path = _path(profile_key) + deadline = time.monotonic() + timeout + while True: + if done(_read(path)): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + with _lock: + _lock.wait(min(remaining, _POLL_SECONDS)) + + +def human_holds(profile_key: Optional[str] = None) -> bool: + return get(profile_key).holder == HUMAN + + +def viewer_may_send_input(viewer_id: str, *, profile_key: Optional[str] = None) -> bool: + lease = get(profile_key) + return lease.holder == HUMAN and lease.viewer_id == viewer_id + + +def assert_agent_may_act(profile_key: Optional[str] = None) -> Lease: + """The lease as of now, or ``HumanHasControl``. Callers keep the returned ``epoch`` and compare it + with ``get().epoch`` after an admitted action: a change means a human took over mid-flight.""" + lease = get(profile_key) + if lease.holder == HUMAN: + raise HumanHasControl( + "A human has taken over this desktop (they may be entering a credential). Screen actions and " + "captures are refused until they hand control back; call computer_use action='wait_for_human' " + "to block until then.") + return lease + + +def _reset_for_tests() -> None: + with _lock: + _listeners.clear() + p = get_hermes_home() / "bot-desktop" / "lease.json" + for f in (p, p.with_suffix(".lock"), p.with_suffix(".json.tmp")): + try: + f.unlink() + except OSError: + pass diff --git a/tools/bot_desktop/rfb_filter.py b/tools/bot_desktop/rfb_filter.py new file mode 100644 index 0000000000000..b4d4a63e2c426 --- /dev/null +++ b/tools/bot_desktop/rfb_filter.py @@ -0,0 +1,100 @@ +"""Byte-level RFB client→server gate for the Bot Desktop WebSocket bridge. + +noVNC's ``viewOnly`` is a UI hint; anyone holding the socket could still inject input. The bridge +parses the client stream and forwards only non-input messages from viewers that do not hold the +lease. RFB messages do not align with WebSocket frames, so this is a stateful stream parser fed +arbitrary chunks (RFC 6143 §7.5 layouts; TigerVNC's EnableContinuousUpdates 150 and Fence 248 pass +through untouched since they carry no input; QEMU Extended KeyEvent 255 is keyboard input — noVNC +switches to it as soon as Xvnc advertises the pseudo-encoding — so it is gated like KeyEvent). + +Xvnc runs ``-SecurityTypes None``, so the handshake is fixed-size: 12-byte version, 1-byte security +choice, then ``ClientInit`` (1 byte). ``ServerInit`` is server→client and never crosses this filter. +""" + +from __future__ import annotations + +from typing import Callable + +_INPUT_TYPES = {4, 5, 6, 255} # KeyEvent, PointerEvent, ClientCutText, QEMU Extended KeyEvent + +# Fixed-length client messages: type -> total length including the type byte. +_FIXED = { + 0: 20, # SetPixelFormat + 3: 10, # FramebufferUpdateRequest + 4: 8, # KeyEvent + 5: 6, # PointerEvent + 150: 10, # EnableContinuousUpdates + 255: 12, # QEMU client message; sub-type 0 = Extended KeyEvent (the only one noVNC sends) +} +_SET_ENCODINGS = 2 +_CLIENT_CUT_TEXT = 6 +_FENCE = 248 + +# TigerVNC's default MaxCutText. The length is client-declared (int32); without a cap a watcher +# with a ticket but no lease could make the bridge buffer ~2 GiB waiting for a payload. +_MAX_CUT_TEXT = 256 * 1024 + + +class RfbClientFilter: + """Feed client bytes with :meth:`feed`; get back the bytes allowed to reach Xvnc. + + ``allow_input`` is consulted per message so a lease flip mid-stream applies to the very next + key or pointer event. + """ + + def __init__(self, allow_input: Callable[[], bool]) -> None: + self._allow_input = allow_input + self._buf = bytearray() + self._handshake_left = 12 + 1 + 1 # version + security type + ClientInit(shared flag) + + def feed(self, chunk: bytes) -> bytes: + self._buf += chunk + out = bytearray() + if self._handshake_left: + take = min(self._handshake_left, len(self._buf)) + if take: + head = bytes(self._buf[:take]) + # ClientInit shared-flag: force shared so a human viewer never disconnects the agent's + # watcher or another observer (Xvnc also runs -AlwaysShared; belt and braces at zero cost). + if self._handshake_left - take == 0 and take >= 1: + head = head[:-1] + b"\x01" + out += head + del self._buf[:take] + self._handshake_left -= take + if self._handshake_left: + return bytes(out) + while self._buf: + length = self._message_length() + if length is None or len(self._buf) < length: + break + msg = bytes(self._buf[:length]) + del self._buf[:length] + if msg[0] in _INPUT_TYPES and not self._allow_input(): + continue + out += msg + return bytes(out) + + def _message_length(self) -> int | None: + t = self._buf[0] + if t in _FIXED: + return _FIXED[t] + if t == _SET_ENCODINGS: + if len(self._buf) < 4: + return None + n = int.from_bytes(self._buf[2:4], "big") + return 4 + 4 * n + if t == _CLIENT_CUT_TEXT: + if len(self._buf) < 8: + return None + n = int.from_bytes(self._buf[4:8], "big", signed=True) + # Extended clipboard (RFB 3.8 + TigerVNC): negative length, |n| bytes follow. + if abs(n) > _MAX_CUT_TEXT: + raise ValueError("clipboard message too large") + return 8 + abs(n) + if t == _FENCE: + if len(self._buf) < 9: + return None + return 9 + self._buf[8] + # Unknown client message: we cannot frame it, and forwarding blind would let an input message + # hide behind it. Drop the rest of the stream; the viewer reconnects. + raise ValueError(f"unknown RFB client message type {t}") diff --git a/tools/bot_desktop/runtime.py b/tools/bot_desktop/runtime.py new file mode 100644 index 0000000000000..a9ce1923f0993 --- /dev/null +++ b/tools/bot_desktop/runtime.py @@ -0,0 +1,385 @@ +"""Bot Desktop runtime: one headless Xfce desktop per Hermes profile, served over RFB on a private +Unix socket, viewed and driven from Hermes Desktop. + +Layout under ``/bot-desktop/``: ``display`` (allocated X display number), ``rfb.sock`` +(Xvnc RFB Unix socket, 0600), ``Xauthority``, ``env`` (DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS +published by the launcher once Xfce's bus exists), ``launcher.pid``, ``launcher.log``, ``xdg/`` +(per-profile XDG_CONFIG_HOME so two profiles never share xfconf). Everything is profile-scoped via +``get_hermes_home()`` so N profiles in one gateway get N desktops: one screen per bot on the shared +machine. + +The launcher is ``launcher.sh`` next to this module; :func:`desktop_env` is what cua-driver and headed +Chromium spawns merge in so the agent acts on this profile's screen and nowhere else. +""" + +from __future__ import annotations + +import contextlib +import logging +import os +import shutil +import signal +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Optional + +from hermes_constants import get_hermes_home + +logger = logging.getLogger(__name__) + +_LAUNCHER = Path(__file__).with_name("launcher.sh") + +# Display numbers below 10 collide with real seats and default Xvfb recipes (:99 is popular too); scan a +# private band and record the choice so restarts reuse it. +_DISPLAY_MIN, _DISPLAY_MAX = 20, 89 + +# Binaries the launcher execs; the package hint is per distro family. +REQUIRED_BINARIES = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session", + "xauth", "xdpyinfo", "setxkbmap", "xprop") + +# Which package in each distro list ships each required binary. Fedora retired the xorg-x11-utils / +# xorg-x11-server-utils umbrellas (per-binary packages since F35) and dnf5 refuses the whole transaction on +# one unknown name, so every binary must map to a package that still resolves; the test suite checks that +# each mapped package is in PACKAGES for its manager. +BINARY_PACKAGES = { + "apt": {"Xvnc": "tigervnc-standalone-server", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", + "xfdesktop": "xfdesktop4", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11", + "xauth": "xauth", "xdpyinfo": "x11-utils", "setxkbmap": "x11-xkb-utils", "xprop": "x11-utils"}, + "dnf": {"Xvnc": "tigervnc-server-minimal", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", + "xfdesktop": "xfdesktop", "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus-x11", + "xauth": "xorg-x11-xauth", "xdpyinfo": "xdpyinfo", "setxkbmap": "setxkbmap", "xprop": "xprop"}, + "pacman": {"Xvnc": "tigervnc", "xfwm4": "xfwm4", "xfce4-panel": "xfce4-panel", "xfdesktop": "xfdesktop", + "xfsettingsd": "xfce4-settings", "dbus-run-session": "dbus", + "xauth": "xorg-xauth", "xdpyinfo": "xorg-xdpyinfo", "setxkbmap": "xorg-setxkbmap", "xprop": "xorg-xprop"}, +} + +PACKAGES = { + "apt": ["tigervnc-standalone-server", "xfce4-panel", "xfwm4", "xfdesktop4", "xfce4-settings", + "xfce4-terminal", "dbus-x11", "x11-xserver-utils", "x11-utils", "x11-xkb-utils", "xauth", + "fonts-dejavu-core"], + "dnf": ["tigervnc-server-minimal", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", + "xfce4-terminal", "dbus-x11", "xsetroot", "xset", "xdpyinfo", "xprop", "xorg-x11-xauth", "setxkbmap", + "dejavu-sans-fonts"], + "pacman": ["tigervnc", "xfce4-panel", "xfwm4", "xfdesktop", "xfce4-settings", "xfce4-terminal", "dbus", + "xorg-xsetroot", "xorg-xset", "xorg-xdpyinfo", "xorg-xprop", "xorg-xauth", "xorg-setxkbmap", + "ttf-dejavu"], +} + + +def state_dir() -> Path: + return get_hermes_home() / "bot-desktop" + + +def is_supported_host() -> bool: + return sys.platform.startswith("linux") + + +def missing_binaries() -> list[str]: + return [b for b in REQUIRED_BINARIES if shutil.which(b) is None] + + +def package_manager() -> Optional[str]: + for pm in ("apt-get", "dnf", "pacman"): + if shutil.which(pm): + return "apt" if pm == "apt-get" else pm + return None + + +def install_command() -> Optional[str]: + pm = package_manager() + if pm is None: + return None + pkgs = " ".join(PACKAGES[pm]) + return { + "apt": f"sudo apt-get install -y --no-install-recommends {pkgs}", + "dnf": f"sudo dnf install -y {pkgs}", + "pacman": f"sudo pacman -S --needed --noconfirm {pkgs}", + }[pm] + + +@dataclass +class DesktopStatus: + profile: str + supported: bool + installed: bool + missing: list[str] + running: bool + pid: Optional[int] + display: Optional[str] + socket: Optional[str] + geometry: str + install_command: Optional[str] + + def as_dict(self) -> Dict[str, object]: + return dict(self.__dict__) + + +def _read(path: Path) -> Optional[str]: + try: + return path.read_text(encoding="utf-8").strip() or None + except OSError: + return None + + +def _pid_alive(pid: int) -> bool: + import psutil + return psutil.pid_exists(pid) + + +def _create_time(pid: int) -> Optional[float]: + import psutil + try: + return psutil.Process(pid).create_time() + except (psutil.Error, OverflowError, ValueError): + return None + + +def _launcher_pid() -> Optional[int]: + """The live launcher's pid, or None. ``launcher.pid`` holds ``" "``: a recycled pid + with a different start time is somebody else's process and must never be reported as ours nor + killed by :func:`stop`. The pre-identity single-number format is treated as not running.""" + raw = _read(state_dir() / "launcher.pid") + pid_s, _, born_s = (raw or "").partition(" ") + if not pid_s.isdigit() or not born_s: + return None + try: + pid, born = int(pid_s), float(born_s) + except ValueError: + return None + actual = _create_time(pid) + return pid if actual is not None and abs(actual - born) < 0.01 else None + + +_X_LOCK_DIR = Path("/tmp") # where X servers write .X-lock (tests point it at a scratch dir) + + +def _display_in_use(num: int) -> bool: + """A live X server owns ``:num``: its lock file names a running pid. A lock left by a crashed + server (dead pid) does not count, so the number can be reclaimed.""" + lock = _X_LOCK_DIR / f".X{num}-lock" + try: + pid = int(lock.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + return False + return _pid_alive(pid) + + +_ALLOC_LOCK = Path("/tmp/.hermes-bot-desktop-alloc.lock") # host-wide: profiles allocate from one band + + +@contextlib.contextmanager +def _flocked(path: Path): + import fcntl # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + with open(path, "a+", encoding="utf-8") as fh: # windows-footgun: ok — Linux-only runtime + fcntl.flock(fh.fileno(), fcntl.LOCK_EX) + try: + yield fh + finally: + fcntl.flock(fh.fileno(), fcntl.LOCK_UN) + + +def _pick_display() -> int: + """Caller holds ``_ALLOC_LOCK``. The recorded number is only reused when no OTHER server holds it now: + after profile A stops, B may have taken A's number, and A's launcher must never unlink B's socket.""" + recorded = _read(state_dir() / "display") + if recorded and recorded.isdigit() and not _display_in_use(int(recorded)): + return int(recorded) + for num in range(_DISPLAY_MIN, _DISPLAY_MAX + 1): + if not _display_in_use(num): + return num + raise RuntimeError("no free X display number in the Bot Desktop band") + + +def _allocate_display() -> int: + with _flocked(_ALLOC_LOCK): + return _pick_display() + + +def desktop_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: + """``base_env`` (default ``os.environ``) with this profile's DISPLAY/XAUTHORITY/DBUS_SESSION_BUS_ADDRESS + merged in when its desktop is running. Unchanged otherwise, so hosts with a real seat keep it. + Pure: never starts anything (it is called from env builders, status probes and tests).""" + env = dict(os.environ if base_env is None else base_env) + published = published_env() + if published: + env.update(published) + env.pop("WAYLAND_DISPLAY", None) # X11 desktop; a leaked Wayland socket flips GTK/Chromium backends + from tools.bot_desktop.browser import env_for_agent + env_for_agent(env) # same binary + user-data-dir as the dock's Browser icon + return env + + +def ensure_started_for_tool() -> None: + """Tool-boundary hook (``computer_use`` dispatch): with ``bot_desktop.auto_start`` (opt-in, default off) a Linux + host that has NO display and the packages installed gets its screen started on first use, so a headless + gateway works the first time instead of answering "no DISPLAY is set". Failure is not an error here; + the tool's own "no display" diagnosis is the right message then.""" + if published_env() or not _should_auto_start(os.environ): + return + try: + start() + except Exception as exc: + logger.info("Bot Desktop auto-start skipped: %s", exc) + + +def _should_auto_start(env: Dict[str, str]) -> bool: + if not is_supported_host() or env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"): + return False + if missing_binaries(): + return False + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly().get("bot_desktop") or {} + return bool(cfg.get("auto_start", False)) + + +def published_env() -> Dict[str, str]: + """Variables the launcher wrote once Xfce's private bus existed; empty when the desktop is down.""" + if _launcher_pid() is None: + return {} + raw = _read(state_dir() / "env") + if not raw: + return {} + out: Dict[str, str] = {} + for line in raw.splitlines(): + key, sep, value = line.partition("=") + if sep: + out[key.strip()] = value.strip() + return out + + +def rfb_socket_path() -> Optional[Path]: + sock = state_dir() / "rfb.sock" + return sock if _launcher_pid() is not None and sock.exists() else None + + +def geometry() -> str: + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly().get("bot_desktop") or {} + return str(cfg.get("geometry") or "1440x900") + + +def status(profile: Optional[str] = None) -> DesktopStatus: + missing: list[str] = missing_binaries() if is_supported_host() else list(REQUIRED_BINARIES) + pid = _launcher_pid() + env = published_env() + return DesktopStatus( + profile=profile or _profile_name(), + supported=is_supported_host(), + installed=not missing, + missing=missing, + running=pid is not None and bool(env.get("DISPLAY")), + pid=pid, + display=env.get("DISPLAY"), + socket=str(rfb_socket_path()) if rfb_socket_path() else None, + geometry=geometry(), + install_command=install_command() if missing else None, + ) + + +def _profile_name() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return get_active_profile_name() or "default" + except Exception: + return "default" + + +def start(*, wait_seconds: float = 15.0) -> DesktopStatus: + """Start this profile's desktop (idempotent). Blocks until the launcher publishes its env file or + ``wait_seconds`` pass; raises ``RuntimeError`` naming the blocker. + + Two locks, both held from the running-check to the launcher's publish: the per-profile ``start.lock`` + so two start() calls for one profile spawn one launcher (the loser sees it running), and the host-wide + display-allocation lock so a second profile cannot pick the same number before this Xvnc has written + ``/tmp/.X-lock`` (it would then fail and its launcher's stale-lock cleanup could remove our socket).""" + if not is_supported_host(): + raise RuntimeError("Bot Desktop runs on Linux gateway hosts only") + missing = missing_binaries() + if missing: + hint = install_command() or "install TigerVNC (Xvnc) and the Xfce core components" + raise RuntimeError(f"Bot Desktop needs {', '.join(missing)} on the gateway host. Install: {hint}") + sd = state_dir() + sd.mkdir(parents=True, exist_ok=True) + os.chmod(sd, 0o700) + with _flocked(sd / "start.lock"): + if _launcher_pid() is not None and published_env().get("DISPLAY"): + return status() + with _flocked(_ALLOC_LOCK): + return _spawn_and_wait(sd, _pick_display(), wait_seconds) + + +def _spawn_and_wait(sd: Path, num: int, wait_seconds: float) -> DesktopStatus: + (sd / "display").write_text(str(num), encoding="utf-8") + env_file = sd / "env" + env_file.unlink(missing_ok=True) + + child_env = {k: v for k, v in os.environ.items() if k not in { + "DISPLAY", "XAUTHORITY", "WAYLAND_DISPLAY", "DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER"}} + child_env.update({ + "HERMES_BD_PROFILE": _profile_name(), + "HERMES_BD_DISPLAY_NUM": str(num), + "HERMES_BD_SOCKET": str(sd / "rfb.sock"), + "HERMES_BD_XAUTH": str(sd / "Xauthority"), + "HERMES_BD_ENV_FILE": str(env_file), + "HERMES_BD_CONFIG_HOME": str(sd / "xdg"), + "HERMES_BD_GEOMETRY": geometry(), + }) + from tools.bot_desktop.browser import dock_command, dock_launch + if (browser := dock_launch()) is not None: + child_env["HERMES_BD_BROWSER_EXEC"] = dock_command(*browser) + # Truncated per start: the log is a diagnostic for THIS launch, and nothing rotates it otherwise. + log = open(sd / "launcher.log", "wb") # noqa: SIM115 — handed to the child, closed by it + proc = subprocess.Popen( # windows-footgun: ok — Linux-only runtime (is_supported_host) + ["bash", str(_LAUNCHER)], env=child_env, stdin=subprocess.DEVNULL, stdout=log, stderr=log, + start_new_session=True, close_fds=True) + log.close() + born = _create_time(proc.pid) + (sd / "launcher.pid").write_text(f"{proc.pid} {born if born is not None else 0}", encoding="utf-8") + + deadline = time.monotonic() + wait_seconds + while time.monotonic() < deadline: + if proc.poll() is not None: + tail = (sd / "launcher.log").read_bytes()[-2000:].decode("utf-8", "replace") + raise RuntimeError(f"Bot Desktop launcher exited with {proc.returncode}:\n{tail}") + if env_file.exists() and (sd / "rfb.sock").exists(): + logger.info("Bot Desktop for profile %s up on :%s", _profile_name(), num) + return status() + time.sleep(0.1) + raise RuntimeError(f"Bot Desktop did not publish its display within {wait_seconds:.0f}s (see {sd / 'launcher.log'})") + + +def stop() -> bool: + """Stop this profile's desktop; True when a running launcher was signalled.""" + if not is_supported_host(): + return False + sd = state_dir() + sd.mkdir(parents=True, exist_ok=True) + with _flocked(sd / "start.lock"): + return _stop_locked(sd) + + +def _stop_locked(sd: Path) -> bool: + pid = _launcher_pid() + if pid is None: + (sd / "env").unlink(missing_ok=True) + return False + # The launcher runs in its own session; killing the group takes Xvnc, dbus and Xfce with it. + try: + os.killpg(pid, signal.SIGTERM) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + except ProcessLookupError: + pass + for _ in range(50): + if not _pid_alive(pid): + break + time.sleep(0.1) + else: + try: + os.killpg(pid, signal.SIGKILL) # windows-footgun: ok — Linux-only runtime (is_supported_host gates start) + except ProcessLookupError: + pass + (sd / "launcher.pid").unlink(missing_ok=True) + (sd / "env").unlink(missing_ok=True) + return True diff --git a/tools/bot_desktop/thumbnail.py b/tools/bot_desktop/thumbnail.py new file mode 100644 index 0000000000000..eeb61b57892b0 --- /dev/null +++ b/tools/bot_desktop/thumbnail.py @@ -0,0 +1,46 @@ +"""Thumbnail of a bot's screen: one JPEG grab of the profile's Xvnc display. + +Feeds the Screen hero in Hermes Desktop (the big preview at the top of a bot's pane). Read-only: +it never touches the lease, so a human in control is not disturbed and the bot is not blocked. +""" + +from __future__ import annotations + +import base64 +import io +import os +import threading +from typing import Optional + +from tools.bot_desktop import runtime + +THUMB_MAX = (960, 600) +_grab_lock = threading.Lock() + + +def thumbnail_data_url(max_size: tuple[int, int] = THUMB_MAX, quality: int = 72) -> Optional[str]: + """``data:image/jpeg;base64,...`` of the running screen, or ``None`` when no screen is up.""" + env = runtime.published_env() + display = env.get("DISPLAY") + if not display or runtime._launcher_pid() is None: + return None + from PIL import ImageGrab # Pillow is a hard dependency; import lazily to keep status calls cheap + + # Xlib reads XAUTHORITY from the process env; the launcher publishes a per-profile cookie file. + # The swap is process-wide, so two profiles grabbed on worker threads at once serialise here or + # one would grab with the other's cookie and restore the wrong value. + with _grab_lock: + previous = os.environ.get("XAUTHORITY") + if env.get("XAUTHORITY"): + os.environ["XAUTHORITY"] = env["XAUTHORITY"] + try: + image = ImageGrab.grab(xdisplay=display) + finally: + if previous is None: + os.environ.pop("XAUTHORITY", None) + else: + os.environ["XAUTHORITY"] = previous + image.thumbnail(max_size) + buf = io.BytesIO() + image.convert("RGB").save(buf, "JPEG", quality=quality, optimize=True) + return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode("ascii") diff --git a/tools/bot_desktop/wallpaper.png b/tools/bot_desktop/wallpaper.png new file mode 100644 index 0000000000000..3d933f827d282 Binary files /dev/null and b/tools/bot_desktop/wallpaper.png differ diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 104bc7694df44..99af9a0a07089 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -42,7 +42,9 @@ def _build_browser_env() -> dict: env = hermes_subprocess_env(inherit_credentials=False) env.update({k: os.environ[k] for k in _BROWSER_PASSTHROUGH_KEYS if k in os.environ}) - return env + # Headed Chromium opens on this profile's Bot Desktop when one is running (human can take it over). + from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env + return _bot_desktop_env(env) try: @@ -817,7 +819,9 @@ def _json_with_fallback(response: Dict[str, Any], result: Dict[str, Any]) -> str def _failed_response(result: Dict[str, Any], default_error: str) -> str: - return _json_with_fallback(_err(result.get("error", default_error)), result) + # ``code`` = machine-readable refusal (human_has_control), same shape as computer_use's. + extra = {"code": result["code"]} if result.get("code") else {} + return _json_with_fallback(_err(result.get("error", default_error), **extra), result) def _tool_response(result: Dict[str, Any], ok: Dict[str, Any], default_error: str) -> str: diff --git a/tools/browser_tool_real_profile.py b/tools/browser_tool_real_profile.py index 103bc7a64cc7f..dfbe6f0e4a025 100644 --- a/tools/browser_tool_real_profile.py +++ b/tools/browser_tool_real_profile.py @@ -163,12 +163,13 @@ def _launch_real_profile_chrome(real_binary: str, copy_dir: str) -> Tuple[Option except OSError: pass chrome_argv = [real_binary, f"--user-data-dir={copy_dir}", *_REAL_PROFILE_CHROME_FLAGS] - _has_display = bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + browser_env = _bt._build_browser_env() # carries the Bot Desktop DISPLAY when one is running + _has_display = bool(browser_env.get("DISPLAY") or browser_env.get("WAYLAND_DISPLAY")) if not (_cloud._is_headed_mode() and (_has_display or not sys.platform.startswith("linux"))): chrome_argv.append("--headless=new") try: chrome_proc = subprocess.Popen(chrome_argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, start_new_session=True, env=_bt._build_browser_env()) + stdin=subprocess.DEVNULL, start_new_session=True, env=browser_env) except (subprocess.SubprocessError, OSError) as e: return None, f"{_RP}the launch failed: {e}" _bt._real_profile_chrome_procs.append(chrome_proc) diff --git a/tools/browser_tool_session.py b/tools/browser_tool_session.py index 4b9b353b3e034..e762d191075ef 100644 --- a/tools/browser_tool_session.py +++ b/tools/browser_tool_session.py @@ -574,6 +574,47 @@ def _run_browser_command( except Exception as e: _bt.logger.warning("Failed to create browser session for task=%s: %s", task_id, e) return {"success": False, "error": f"Failed to create browser session: {str(e)}"} + # The bot's LOCAL browser lives on its Bot Desktop screen, in the same profile a human who took + # over is typing into. While the human holds the lease every action AND read against it is + # refused (the page may show their credential); the fence brackets the whole run so a takeover + # mid-command also voids the result. Cloud / user-supplied CDP sessions are a different browser. + if _shares_bot_desktop_browser(session_info): + from tools.bot_desktop import lease as _bd_lease + try: + admitted = _bd_lease.assert_agent_may_act() + except _bd_lease.HumanHasControl as e: + return {"success": False, "error": str(e), "code": "human_has_control"} + result = _run_browser_command_unfenced(task_id, command, args, timeout, _engine_override, browser_cmd, session_info) + if _bd_lease.get().epoch != admitted.epoch: + return {"success": False, "code": "human_has_control", + "error": "A human took over the bot's screen while this browser command ran; its result was " + "discarded. Call computer_use action='wait_for_human' to block until they hand back."} + return result + return _run_browser_command_unfenced(task_id, command, args, timeout, _engine_override, browser_cmd, session_info) + + +def _shares_bot_desktop_browser(session_info: Dict[str, Any]) -> bool: + """Decided by provenance, not transport: every LOCAL session (plain ``--session``, real-profile CDP + attach, Lightpanda) is a browser Hermes launched with this profile's Bot Desktop DISPLAY, so it is the + screen a human who took over is typing into. Cloud / user-supplied CDP sessions are another browser. + A human lease with the screen already gone (dead Xvnc) still fences — computer_use does the same.""" + if not (session_info.get("features") or {}).get("local"): + return False + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + return bool(_bd_runtime.published_env().get("DISPLAY")) or _bd_lease.human_holds() + + +def _bot_desktop_attach_port(session_info: Dict[str, Any]) -> Optional[int]: + """DevTools port of a human-started Chromium on the Bot Desktop's shared profile, else ``None``.""" + if not _shares_bot_desktop_browser(session_info): + return None + from tools.bot_desktop import browser as _bd_browser + return _bd_browser.running_instance_cdp_port(str(_bd_browser.profile_dir()), + exclude_session=session_info["session_name"]) + + +def _run_browser_command_unfenced(task_id: str, command: str, args: List[str], timeout: int, + _engine_override: Optional[str], browser_cmd, session_info: Dict[str, Any]) -> Dict[str, Any]: # Cleanup stops the supervisor before closing the backend; keep it stopped. if command != "close" and session_info.get("cdp_url"): _cdp._ensure_cdp_supervisor(task_id) @@ -587,6 +628,12 @@ def _run_browser_command( backend_args = ["--cdp", session_info["cdp_url"]] else: backend_args = ["--session", session_info["session_name"]] + if (bd_port := _bot_desktop_attach_port(session_info)) is not None: + # A Chromium already runs on the Bot Desktop's shared profile (the human clicked the dock's + # Browser first): a launch would be forwarded into it by Chromium's singleton and die without + # a DevTools endpoint, so the session's daemon attaches to the port it advertises instead. + # Same daemon (keyed by --session) either way, so snapshot refs stay valid across commands. + backend_args += ["--cdp", str(bd_port)] if _cloud._is_headed_mode(): backend_args.append("--headed") if engine != "auto" and not _bt._is_camofox_mode(): diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index 9be4b19cb77ea..d6075d2e7aab1 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -100,12 +100,28 @@ def _computer_use_max_image_dimension() -> Optional[int]: dim = 1456 return dim if dim > 0 else None +def desktop_identity(env: Optional[Dict[str, str]] = None) -> str: + """The screen a backend spawned from ``env`` acts on: its DISPLAY (``''`` when none). Recorded next to the + cached backend so a Bot Desktop that starts (or restarts on another number) AFTER the backend was cached is + noticed — the cached cua-driver still points at the old seat or at no display at all.""" + return str((cua_driver_child_env(env) if env is None else env).get("DISPLAY") or "") + + +def backend_display_stale(recorded: str, current: str) -> bool: + """True when a cached backend's recorded display identity no longer matches the one a fresh spawn would get.""" + return (recorded or "") != (current or "") + + def cua_driver_child_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: """Env for spawning cua-driver: ``base_env`` (default ``os.environ``) plus ``CUA_DRIVER_RS_TELEMETRY_ENABLED=0`` unless the user opted in, plus the native-Wayland bridge (``computer_use.native_wayland`` config opt-in, only when the child has a Wayland display). Used by every spawn site (MCP, status, doctor, install) so CLI and gateway runtimes share one policy.""" env = dict(os.environ if base_env is None else base_env) + # A running Bot Desktop for this profile owns the agent's screen: DISPLAY/XAUTHORITY/DBUS point there so + # cua-driver never acts on a seat the human is sitting at (#90374 class) and headless hosts get a display. + from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env + env = _bot_desktop_env(env) if _cua_telemetry_disabled(): env[_CUA_TELEMETRY_ENV_VAR] = "0" if sys.platform == "linux" and env.get("WAYLAND_DISPLAY") and bool(_computer_use_cfg().get("native_wayland", False)): diff --git a/tools/computer_use/handoff.py b/tools/computer_use/handoff.py new file mode 100644 index 0000000000000..bdd38acbb1ad0 --- /dev/null +++ b/tools/computer_use/handoff.py @@ -0,0 +1,50 @@ +"""Human handoff actions for ``computer_use`` on a Bot Desktop: ``request_handoff`` asks the person to +take over the screen (login, 2FA, CAPTCHA, payment) and ``wait_for_human`` blocks until control is +back. Both are answered without touching cua-driver, so a human typing a credential is never +captured. + +The Desktop pane shows the request (the gateway broadcasts the lease change). Reaching the person +anywhere else is the model's job: it relays the ask in its reply, which is what lands in the chat +surface the user is actually on. This module only records intent on the lease and waits. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict + +from tools.bot_desktop import lease as _lease + +HANDOFF_ACTIONS = frozenset({"request_handoff", "wait_for_human"}) +_DEFAULT_WAIT_SECONDS = 600.0 +_MAX_WAIT_SECONDS = 1800.0 +_DEFAULT_GRACE_SECONDS = 60.0 + + +def handle_handoff(action: str, args: Dict[str, Any]) -> str: + if action == "request_handoff": + reason = str(args.get("reason") or "The agent needs you to complete a step on its screen.").strip() + _lease.request_handoff(reason) + return json.dumps({ + "ok": True, "action": action, "state": _lease.get().as_dict(), + "next": "Hermes Desktop now shows 'Bot needs you' on this bot's Screen. Tell the user in your " + "reply what to do and that they can take over from Bots > Screen, then call " + "computer_use action='wait_for_human' to block until they hand control back and " + "re-capture before continuing — the screen state is whatever they left."}) + timeout = min(_MAX_WAIT_SECONDS, max(1.0, float(args.get("seconds") or _DEFAULT_WAIT_SECONDS))) + grace = min(timeout, max(0.0, float(args.get("grace") or _DEFAULT_GRACE_SECONDS))) + # Nobody has taken over yet: give them `grace` to click Take over, then return so the model can chase + # the user in chat instead of blocking the whole timeout on a request nobody saw. Once a human holds + # the screen, wait the full timeout for the hand-back. + if not _lease.wait_for_takeover_or_release(timeout=grace): + state = _lease.get().as_dict() + return json.dumps({"ok": False, "action": action, "code": "no_takeover", "state": state, + "error": f"Nobody took over within {grace:.0f}s. Ask the user in chat to open Bots > Screen and " + "click Take over, then call wait_for_human again."}) + released = _lease.wait_for_release(timeout=timeout) + state = _lease.get().as_dict() + if released: + return json.dumps({"ok": True, "action": action, "state": state, + "next": "Control is back with you. Take a fresh capture; do not assume prior state."}) + return json.dumps({"ok": False, "action": action, "code": "still_waiting", "state": state, + "error": f"No hand-back within {timeout:.0f}s. Call wait_for_human again or ask the user in chat."}) diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index a38943c142bba..880d9bdc58273 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -32,12 +32,16 @@ "list_apps", "list_windows", "focus_app", + "request_handoff", + "wait_for_human", ], "description": ( "Which action to perform. `capture` is free (no side effects). All other actions " "require approval unless auto-approved. Use `set_value` for select/popup elements and " "sliders — it selects the matching option directly without opening the native menu (no " - "focus steal)." + "focus steal). When a login, 2FA, CAPTCHA or payment step needs the human, call " + "`request_handoff` (with `reason`) so they can take over this screen from the Hermes " + "Desktop app, then `wait_for_human`; while they hold control every other action is refused." ), }, "mode": { @@ -141,13 +145,15 @@ ), }, "text": {"type": "string", "description": "Text to type (respects the current layout)."}, + "reason": {"type": "string", "description": "request_handoff: one sentence telling the human what to do on the screen (e.g. 'Sign in to LinkedIn and complete 2FA')."}, "keys": { "type": "string", "description": ( "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine." ), }, - "seconds": {"type": "number", "description": "Seconds to wait. Max 30."}, + "seconds": {"type": "number", "description": "wait: seconds to pause (max 30). wait_for_human: how long to block for the hand-back (default 600, max 1800)."}, + "grace": {"type": "number", "description": "wait_for_human: seconds to wait for someone to take over before returning no_takeover (default 60); once a human holds control the full `seconds` applies."}, "raise_window": { "type": "boolean", "description": ( diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index afb000913f173..32f892db38c16 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -79,6 +79,7 @@ def _input_target_mismatch(backend, requested_app: str) -> Optional[str]: _backends: Dict[str, ComputerUseBackend] = {} _backend_call_locks: Dict[str, threading.RLock] = {} _backend_permission_modes: Dict[str, str] = {} +_backend_displays: Dict[str, str] = {} # DISPLAY the cached backend was spawned against (Bot Desktop rebind) # (home key, provider, model) → bool. The decision reads the active profile's config (auxiliary.vision # override, declared supports_vision), so a multiplexed process must not serve profile A's verdict to B. _AUX_VISION_ROUTE_CACHE: Dict[Tuple[str, str, str], bool] = {} @@ -133,7 +134,9 @@ def _install_backend(sid: str, backend: ComputerUseBackend, permission_mode: str """Record a backend in the session caches (the empty session also mirrors it onto the ``_backend`` hook). Caller holds ``_backend_lock``.""" global _backend + from tools.computer_use.cua_backend import desktop_identity _backends[sid], _backend_permission_modes[sid] = backend, permission_mode + _backend_displays[sid] = desktop_identity() _backend_call_locks[sid] = threading.RLock() _backend = backend if sid == "" else _backend return backend @@ -142,7 +145,7 @@ def _detach_locked(sid: str) -> Tuple[Optional[ComputerUseBackend], Optional[thr """Remove one session's cache entries, plus the ``_backend`` injection hook when it aliases the empty session (older callers/tests may populate only the hook). Caller holds ``_backend_lock``.""" global _backend - _backend_permission_modes.pop(sid, None) + _backend_permission_modes.pop(sid, None), _backend_displays.pop(sid, None) backend, call_lock = _backends.pop(sid, None), _backend_call_locks.pop(sid, None) if sid == "": backend = _backend if backend is None else backend @@ -170,9 +173,12 @@ def _get_backend(session_id: str = "") -> ComputerUseBackend: backend = _new_backend(permission_mode) backend.start() # under the cache lock: one backend per session; a concurrent toggle releases it return _install_backend(sid, backend, permission_mode) - if _backend_permission_modes.get(sid, "standard") == permission_mode: + from tools.computer_use.cua_backend import backend_display_stale, desktop_identity + if (_backend_permission_modes.get(sid, "standard") == permission_mode + and not backend_display_stale(_backend_displays.get(sid, ""), desktop_identity())): return cached - # Cua's mode is immutable after daemon startup: a /yolo toggle replaces only this session's backend. + # Cua's mode and DISPLAY are fixed at daemon startup: a /yolo toggle, or a Bot Desktop that started + # (or moved) after this backend was cached, replaces only this session's backend. _, stale_lock = _detach_locked(sid) # stopped outside the cache lock; the loop re-reads the mode first _stop_backend(cached, stale_lock, lambda e: None) @@ -207,7 +213,7 @@ def _shutdown_backend_atexit() -> None: if _backend is not None: unique.setdefault(id(_backend), (_backend, _backend_call_locks.get(""))) _backend = None - _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear() + _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear(), _backend_displays.clear() with _approval_lock: _session_auto_approve.clear(), _always_allow.clear(), _escalation_warned.clear() for backend, call_lock in unique.values(): @@ -248,6 +254,19 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: if not action: return json.dumps({"error": "missing `action`"}) session_id = str(kwargs.get("session_id") or "") # approval-state / daemon-mode isolation key + from tools.computer_use.handoff import HANDOFF_ACTIONS, handle_handoff + if action in HANDOFF_ACTIONS: + return handle_handoff(action, args) + # Bot Desktop lease: while a human drives the screen every action, capture included, is refused. + from tools.bot_desktop import lease as _bd_lease + from tools.bot_desktop.runtime import ensure_started_for_tool as _bd_ensure_started + def _refused(e: Exception) -> str: + return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)}) + try: + admitted = _bd_lease.assert_agent_may_act() + except _bd_lease.HumanHasControl as e: + return _refused(e) + _bd_ensure_started() # headless gateway: bring the profile's screen up before the backend probes DISPLAY if (err := _reject_unsafe(action, args)) is not None: return err scopes = ([action] if action in _ACTIONS and _ACTIONS[action].destructive else []) + ( @@ -265,7 +284,27 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: with _backend_lock: call_lock = _backend_call_locks.setdefault(session_id, threading.RLock()) with call_lock: - return _dispatch(backend, action, args) + # Re-check under the dispatch lock: approval, backend start-up and lock waits above can take + # seconds, and a human may have taken over meanwhile. A result produced after such a flip is + # discarded too — it may picture what they typed. + try: + _bd_lease.assert_agent_may_act() + except _bd_lease.HumanHasControl as e: + return _refused(e) + + def _fence() -> None: + # Any lease transition since admission voids the frame — including a full take-over / + # hand-back cycle that already finished: it still belongs to the human's turn. Capture + # paths call this BEFORE the frame is persisted, spilled or sent to auxiliary vision. + if _bd_lease.get().epoch != admitted.epoch: + raise _bd_lease.HumanHasControl( + "A human took over this desktop while the action ran; its result was discarded. " + "Re-capture (or call computer_use action='wait_for_human' if they still hold control).") + result = _dispatch(backend, action, args, fence=_fence) + _fence() + return result + except _bd_lease.HumanHasControl as e: + return _refused(e) except Exception as e: logger.exception("computer_use %s failed", action) return json.dumps({"error": f"{action} failed: {e}"}) @@ -334,12 +373,13 @@ def _do_scroll(backend, action, args, **delivery): return backend.scroll(direction=args.get("direction", "down"), amount=int(args.get("amount", 3)), element=args.get("element"), **_scroll_xy(args), modifiers=args.get("modifiers"), **delivery) -def _do_capture(backend, action, args, **_): +def _do_capture(backend, action, args, fence=lambda: None, **_): if (mode := str(args.get("mode", "som"))) not in {"som", "vision", "ax"}: return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"}) # pid/window_id forwarded only when given so older backends keep their defaults. - return _capture_response(backend.capture(mode=mode, app=args.get("app"), - **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None})) + cap = backend.capture(mode=mode, app=args.get("app"), **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None}) + fence() + return _capture_response(cap) def _do_listing(backend, action, args, key, **_): return json.dumps({key: (items := getattr(backend, action)()), "count": len(items)}) @@ -389,7 +429,9 @@ def _summarize_click(action: str, args: Dict[str, Any], fg: str) -> str: "input_text": "type", "screenshot": "capture", "get_window_state": "capture", "left_click": "click", "mouse_click": "click", } -def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> Any: +def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any], fence: Callable[[], None] = lambda: None) -> Any: + """``fence`` raises when the screen lease moved since admission; capture paths call it as soon as the + frame is in hand, before anything derived from it leaves the process.""" spec = _ACTIONS.get(action) if spec is None: return json.dumps({"error": f"unknown action {action!r}" + (f" — did you mean {hint!r}? See the action enum in the tool schema." @@ -402,10 +444,11 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> f"{action} would go to the current target {mismatch!r}, not {requested_app.strip()!r} " "— input actions always hit the sticky target from the last capture/focus_app. " f"Call capture(app={requested_app.strip()!r}) or focus_app first, then retry.")}) - # delivery_mode / bring_to_front thread through every input action (background → foreground ladder). - res = spec.handler(backend, action, args, delivery_mode=args.get("delivery_mode"), - bring_to_front=bool(args.get("bring_to_front"))) - return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after"))) + # delivery_mode / bring_to_front thread through every input action (background → foreground ladder); + # read-only actions get the lease fence instead (delivery kwargs would leak into backend input calls). + res = spec.handler(backend, action, args, **(dict(delivery_mode=args.get("delivery_mode"), bring_to_front=bool(args.get("bring_to_front"))) + if spec.input else dict(fence=fence))) + return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after")), fence) # ── Response shaping ──────────────────────────────────────────────────────── def _classify_action_result(res: ActionResult) -> Dict[str, Any]: @@ -583,7 +626,8 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME "elements_file — read_file/search_files it, or pass app= to narrow scope)") return _text_capture_payload(v, "\n".join(lines), extra) -def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool) -> Any: +def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool, + fence: Callable[[], None] = lambda: None) -> Any: # No follow-up capture after a failed action: a normal-looking screenshot would suggest success. if not do_capture or not res.ok: return _text_response(res) @@ -596,6 +640,7 @@ def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_cap except Exception as e: logger.warning("follow-up capture failed: %s", e) return _text_response(res) + fence() resp, payload = _capture_response(cap), _action_payload(res) if isinstance(resp, dict) and resp.get("_multimodal"): # Keep the evidence/verdict contract visible alongside the image — it governs whether input may repeat. diff --git a/tui_gateway/methods_display.py b/tui_gateway/methods_display.py new file mode 100644 index 0000000000000..660a8a685ff1f --- /dev/null +++ b/tui_gateway/methods_display.py @@ -0,0 +1,221 @@ +"""Bot Desktop JSON-RPC handlers: the Desktop app's door to a profile's headless screen. + +``display.status`` reports runtime + lease; ``display.start`` / ``display.stop`` manage the Xvnc/Xfce +process; ``display.observe`` mints a single-use ticket the renderer redeems on ``/api/display/ws`` +(``hermes_cli.web_routers.display``) to stream raw RFB; ``display.lease.acquire`` / ``release`` are +Take over / Hand back. ``display.install`` runs the distro package install on the gateway host: sudo +privilege is asked for through the masked ``display.install.sudo.request`` card (same ``_block`` bridge as +the terminal tool's sudo prompt), stdout streams as ``display.install.log`` and the run ends with +``display.install.done`` carrying a fresh status snapshot. Every handler is profile-scoped so a multiplexed gateway answers for the bot +the pane is looking at. Lease transitions fan out as the global ``display.lease`` event so every +connected client repaints (badge on the bot row, red border on the viewer, agent handoff prompt). + +Bodies are rebound onto server.py's globals (method_ctx.bind_module) and reference them bare. +""" + +import logging +import threading +import weakref + +from .method_ctx import HandlerRegistry, bind_module + +logger = logging.getLogger(__name__) +_registry = HandlerRegistry() +method = _registry.method +_profile_scoped = _registry.profile_scoped + +_DISPLAY_ERR = 5300 +_lease_listener_installed = threading.Event() + + +def _lease_view(lease) -> dict: + """The lease as clients may see it: the holder's viewer id is a capability (whoever presents it + co-drives or releases the lease), so it is replaced by a short hash the holder can match against + its own id to know it is the one in control.""" + import hashlib + d = lease.as_dict() + vid = d.pop("viewer_id") + d["viewer_id"] = None + d["viewer_hash"] = hashlib.sha256(vid.encode()).hexdigest()[:12] if vid else None + return d + + +def _display_snapshot() -> dict: + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + st = _bd_runtime.status() + return {**st.as_dict(), "lease": _lease_view(_bd_lease.get()), "profile_key": hermes_home_key()} + + +def _install_lease_listener() -> None: + """Once per process: broadcast every lease change to all connected clients.""" + if _lease_listener_installed.is_set(): + return + from tools.bot_desktop import lease as _bd_lease + + def _on_change(profile_key: str, lease) -> None: + _broadcast_global_event("display.lease", {"profile_key": profile_key, "lease": _lease_view(lease)}) + _bd_lease.on_change(_on_change) + _lease_listener_installed.set() # only once the subscription exists, or a failed import would silence every client + + +@method("display.status") +@_profile_scoped +def _(rid, params: dict) -> dict: + _install_lease_listener() + try: + return _ok(rid, _display_snapshot()) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.thumbnail") +@_profile_scoped +def _(rid, params: dict) -> dict: + """One JPEG grab of the bot's screen (``data_url``: null while stopped). Read-only: no lease change. + Suppressed while a human holds the lease — the frame may show what they are typing.""" + try: + from tools.bot_desktop import lease as _bd_lease + if _bd_lease.human_holds(): + return _ok(rid, {"data_url": None, "suppressed": "human_has_control"}) + from tools.bot_desktop.thumbnail import thumbnail_data_url + return _ok(rid, {"data_url": thumbnail_data_url()}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.start") +@_profile_scoped +def _(rid, params: dict) -> dict: + _install_lease_listener() + from tools.bot_desktop import runtime as _bd_runtime + try: + _bd_runtime.start() + return _ok(rid, _display_snapshot()) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.stop") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime + try: + _bd_lease.release() + stopped = _bd_runtime.stop() + return _ok(rid, {**_display_snapshot(), "stopped": stopped}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +# viewer ids minted per connection (keyed by the transport that asked), so a reconnecting pane can +# keep its identity — and its lease — while nobody can claim an id minted for another connection. +_minted_viewer_ids: "weakref.WeakKeyDictionary[object, set[str]]" = weakref.WeakKeyDictionary() + + +def _mint_viewer_id(requested: str) -> str: + """Server-minted viewer identity. ``requested`` is honoured only when THIS connection minted it + earlier; anything else (including a holder id read off display.status) gets a fresh id.""" + import secrets + try: + mine = _minted_viewer_ids.setdefault(current_transport(), set()) + except TypeError: # stdio / slotted transports cannot be weakly referenced: always mint + mine = set() + if requested in mine: + return requested + viewer_id = secrets.token_urlsafe(16) + mine.add(viewer_id) + return viewer_id + + +@method("display.observe") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Mint a single-use, 30 s ticket for ``/api/display/ws``. The ticket carries the profile home so + the bridge dials THIS profile's RFB socket, and a server-minted viewer id (returned to the caller, + who passes it to ``display.lease.acquire`` / ``release``) so the lease can name the holder.""" + from hermes_constants import get_hermes_home + from hermes_cli.dashboard_auth.ws_tickets import mint_ticket + from tools.bot_desktop import runtime as _bd_runtime + try: + if _bd_runtime.rfb_socket_path() is None: + return _err(rid, _DISPLAY_ERR, "this profile's Bot Desktop is not running; call display.start first") + viewer_id = _mint_viewer_id(str(params.get("viewer_id") or "").strip()) + ticket = mint_ticket(user_id=f"display:{viewer_id}", provider="bot-desktop", + extra={"hermes_home": str(get_hermes_home()), "viewer_id": viewer_id}) + return _ok(rid, {"ticket": ticket, "path": "/api/display/ws", "viewer_id": viewer_id, + **_display_snapshot()}) + except Exception as e: + return _err(rid, _DISPLAY_ERR, str(e)) + + +@method("display.install") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Start the package install in the background; the renderer follows ``display.install.log`` / + ``display.install.done``. Refused while one is already running for this profile.""" + from hermes_constants import hermes_home_key + from tools.bot_desktop import install as _bd_install, runtime as _bd_runtime + if not _bd_runtime.is_supported_host(): + return _err(rid, _DISPLAY_ERR, "Bot Desktop runs on Linux gateway hosts only") + if _bd_runtime.install_command() is None: + return _err(rid, _DISPLAY_ERR, "no supported package manager (apt-get, dnf, pacman) on this host") + profile_key = hermes_home_key() + sid = str(params.get("session_id") or "") + + def _ask_password() -> str: + return _block("display.install.sudo.request", sid, {"profile_key": profile_key}, timeout=300) + + def _line(text: str) -> None: + _broadcast_global_event("display.install.log", {"profile_key": profile_key, "line": text}) + + # The worker thread inherits NO context: the caller's transport (so the sudo card reaches the + # CLIENT that clicked Install) and the profile scope `_profile_scoped` installed (so status, lock + # and events all speak for the requested profile) are carried across with copy_context(). + import contextvars + ctx = contextvars.copy_context() + + def _run() -> None: + try: + code = _bd_install.install_packages(ask_password=_ask_password, on_line=_line, claimed=True) + except Exception as e: + _line(f"install failed: {e}") + code = 1 + _broadcast_global_event("display.install.done", {"profile_key": profile_key, "code": code, + "status": _display_snapshot()}) + + try: + _bd_install.claim() # atomic: two fast clicks cannot both start a package manager + except _bd_install.InstallBusy as e: + return _err(rid, _DISPLAY_ERR, str(e)) + threading.Thread(target=ctx.run, args=(_run,), name=f"bot-desktop-install:{profile_key}", daemon=True).start() + return _ok(rid, {"started": True, "command": _bd_runtime.install_command(), "profile_key": profile_key}) + + +@method("display.lease.acquire") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease + viewer_id = str(params.get("viewer_id") or "").strip() + if not viewer_id: + return _err(rid, _DISPLAY_ERR, "viewer_id required") + lease = _bd_lease.acquire(viewer_id, reason=str(params.get("reason") or "")) + return _ok(rid, {"lease": _lease_view(lease)}) + + +@method("display.lease.release") +@_profile_scoped +def _(rid, params: dict) -> dict: + from tools.bot_desktop import lease as _bd_lease + viewer_id = str(params.get("viewer_id") or "").strip() or None + # lease.release(None) skips the holder check; a client that lost its viewer id must not be able to + # yank control from whoever holds it unless it says so explicitly (force). + if viewer_id is None and not params.get("force") and _bd_lease.human_holds(): + return _err(rid, _DISPLAY_ERR, "viewer_id required to release another viewer's lease (or pass force: true)", + data={"code": "viewer_mismatch"}) + lease = _bd_lease.release(viewer_id) + return _ok(rid, {"lease": _lease_view(lease)}) + + +def register(server) -> None: + bind_module(globals(), server, skip=("_",)) diff --git a/tui_gateway/methods_display_watch.py b/tui_gateway/methods_display_watch.py new file mode 100644 index 0000000000000..3050d08b7678e --- /dev/null +++ b/tui_gateway/methods_display_watch.py @@ -0,0 +1,87 @@ +"""Cross-process lease watcher: ``display.lease`` for transitions made OUTSIDE ``hermes serve``. + +The lease (``tools.bot_desktop.lease``) lives on disk and is changed by whatever process hosts the +agent — the messaging gateway's ``request_handoff``, a ``hermes chat`` takeover, a cron worker's +release. ``lease.on_change`` only fires in the writing process, so ``methods_display``'s in-process +listener never sees those; the Desktop's "Bot needs you" badge and hero tone stayed stale until the +pane was reopened. One daemon thread stats every served home's ``bot-desktop/lease.json`` (launch +home + ``_served_profile_homes``) every 0.5s and broadcasts the SAME ``display.lease`` payload when +the epoch moves. Bodies are rebound onto server.py's globals (method_ctx.bind_module). +""" + +from __future__ import annotations + +import threading +import time +from pathlib import Path + +from .method_ctx import bind_module + +_LEASE_POLL_S = 0.5 +_lease_watcher_started = threading.Event() +# profile key → last epoch broadcast or seen (in-process transitions record theirs too, so a change +# made by THIS process is not re-broadcast when its file write is noticed a tick later). +_lease_epochs: dict[str, int] = {} +_lease_mtimes: dict[str, int | None] = {} + + +def _lease_event_payload(profile_key: str, lease) -> dict: + # Same shape and the same redaction (viewer_hash, never the raw id) as the in-process broadcast. + from tui_gateway.methods_display import _lease_view + return {"profile_key": profile_key, "lease": _lease_view(lease)} + + +def _watched_lease_homes() -> list[Path]: + return [Path(_hermes_home), *_served_profile_homes] + + +def _poll_lease_files() -> None: + """One pass: read a home's lease only when its file mtime moved; broadcast when the epoch did.""" + from hermes_constants import hermes_home_key + from tools.bot_desktop import lease as _bd_lease + for home in _watched_lease_homes(): + key = hermes_home_key(home) + try: + mtime = (home / "bot-desktop" / "lease.json").stat().st_mtime_ns + except OSError: + mtime = None + if mtime == _lease_mtimes.get(key, 0): + continue + _lease_mtimes[key] = mtime + lease = _bd_lease.get(str(home)) + if key not in _lease_epochs: # first sighting seeds silently: display.status carries it + _lease_epochs[key] = lease.epoch + continue + if lease.epoch == _lease_epochs[key]: + continue + _lease_epochs[key] = lease.epoch + _broadcast_global_event("display.lease", _lease_event_payload(key, lease)) + + +def _ensure_lease_watcher() -> None: + """Once per process, from the first display.* call: start the lease-file poll thread and mark + in-process transitions as seen so they broadcast exactly once (via the in-process listener).""" + if _lease_watcher_started.is_set(): + return + _lease_watcher_started.set() + from tools.bot_desktop import lease as _bd_lease + + def _seen_locally(profile_key: str, lease) -> None: + # methods_display's listener broadcasts in-process transitions; only then is the file + # move ours to skip. Before display.status installed it, the poll below carries them. + if _lease_listener_installed.is_set(): + _lease_epochs[profile_key] = lease.epoch + _bd_lease.on_change(_seen_locally) + + def _loop() -> None: + while True: + try: + _poll_lease_files() + except Exception: # noqa: BLE001 - a torn read must not kill the watcher + logger.debug("lease watcher poll failed", exc_info=True) + time.sleep(_LEASE_POLL_S) + threading.Thread(target=_loop, name="hermes-lease-watcher", daemon=True).start() + + +def register(server) -> None: + bind_module(globals(), server, skip=("_",)) diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index bdb0a40b929f0..8824dc3b23c6a 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -1113,7 +1113,7 @@ def _(rid, params: dict) -> dict: _LATE_RESPOND_KEYS = { "terminal.read.respond": "text", "preview.read.respond": "text", "preview.act.respond": "text", "window.read.respond": "text", "tour.respond": "text", "mcp.setup.respond": "result", - "sudo.respond": "password", "secret.respond": "value", "vault.unlock.respond": "password", + "sudo.respond": "password", "display.install.sudo.respond": "password", "secret.respond": "value", "vault.unlock.respond": "password", "vault.save_login.respond": "login", "vault.code.respond": "code"} for _name, _key in _LATE_RESPOND_KEYS.items(): method(_name)(lambda rid, params, _k=_key: _respond(rid, params, _k, allow_expired=True)) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index cbfe41572dbc9..b5101afc0ba2e 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -1252,7 +1252,7 @@ def _enable_gateway_prompts() -> None: # Blocking bridges whose `*.respond` tolerates a late reply (allow_expired=True): on timeout the tool # returns empty, but a slow renderer could still answer and hit a raw 4009 — `.expire` tears the card down. _EXPIRING_REQUESTS = frozenset({ - "secret.request", "sudo.request", "vault.unlock.request", "vault.save_login.request", "vault.code.request", "clarify.request", + "secret.request", "sudo.request", "display.install.sudo.request", "vault.unlock.request", "vault.save_login.request", "vault.code.request", "clarify.request", "terminal.read.request", "preview.read.request", "preview.act.request", "window.read.request", "mcp.setup.request", "tour.request", @@ -3230,7 +3230,8 @@ def _resolve_name(name: str) -> str: methods_projects as _methods_projects, methods_session_foreign as _methods_session_foreign, methods_session_control as _methods_session_control, methods_subagents as _methods_subagents, methods_vault as _methods_vault, methods_free_tier as _methods_free_tier, - methods_connectors as _methods_connectors) + methods_connectors as _methods_connectors, methods_display as _methods_display, + methods_display_watch as _methods_display_watch) for _m in ( _session_transports, _session_reaper, _session_lifecycle, _session_workdir, _compute_host_bridge, _model_switch, @@ -3240,6 +3241,7 @@ def _resolve_name(name: str) -> str: _methods_browser_control, _methods_session, _methods_prompt, _methods_config, _methods_config_set, _methods_complete, _methods_tools, _methods_profiles, _methods_images, _methods_bot_relay, _prompt_turn, _billing_view, _methods_projects, _methods_session_foreign, - _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors): + _methods_session_control, _methods_subagents, _methods_vault, _methods_free_tier, _methods_connectors, + _methods_display, _methods_display_watch): _m.register(sys.modules[__name__]) del _m diff --git a/tui_gateway/ws.py b/tui_gateway/ws.py index 480ed36636940..448224fab1a86 100644 --- a/tui_gateway/ws.py +++ b/tui_gateway/ws.py @@ -304,6 +304,7 @@ def _error(code: int, message: str, req_id: Any) -> dict: # Live-apply skins Hermes activates mid-conversation, and track this peer for session-less # global broadcasts write_json can't route. server._ensure_skin_watcher() + server._ensure_lease_watcher() # cross-process lease moves → display.lease server.register_live_transport(transport) # Cross-backend liveness: a heartbeat row lets the startup orphan sweep tell "live but idle # backend" from "truly orphaned". Idempotent and once-per-process, like the orphan sweep (the diff --git a/website/docs/user-guide/features/bot-screen.md b/website/docs/user-guide/features/bot-screen.md new file mode 100644 index 0000000000000..f201fbf775ee1 --- /dev/null +++ b/website/docs/user-guide/features/bot-screen.md @@ -0,0 +1,168 @@ +--- +title: Bot Screen +sidebar_position: 17 +--- + +# Bot Screen + +On a headless Linux gateway host (a server, a cloud VM, Hermes Cloud) each bot +gets its **own desktop**: an Xfce screen the bot's `computer_use` and headed +browser act on, streamed live into Hermes Desktop. Watch what the bot does, +**take over** when it hits a login, 2FA prompt, CAPTCHA or payment step, then +**hand control back** and let it continue with the session you just signed in +to. The bot keeps working after you close the app or turn off your laptop; the +screen lives on the gateway host, not on your machine. + +Every Hermes profile ("bot") has its own screen, its own browser profile and +its own cookies. Screens are work surfaces, not security boundaries: the bots +share the host's user account, files and network (the same model as other +hosted-agent products). + +**Threat model.** The screen's RFB socket, the X display, the browser profile +and the control-lease file all belong to the gateway's OS user. Any process +running as that user — another bot on the same host, and the bot's own +`terminal` tool included — can reach them directly, bypassing the pane and the +lease. The lease is a tool-level fence on `computer_use` and the browser tools, +not an OS one. Running each bot as its own OS user is out of scope; if that +isolation matters to you, put the bots on separate hosts. + +## Requirements + +- The gateway host runs Linux. macOS and Windows hosts already have a real + display; the pane is not offered there. +- TigerVNC's `Xvnc` and the Xfce core components are installed on the host. + Nothing installs them silently: `hermes update` and fresh installs leave every + machine as it is. When they are missing the Screen pane in Hermes Desktop shows + **Install on host** — one click runs the package manager on the gateway host + (it asks for that host's sudo password in a masked card; the password goes to + that host only and is never stored) and streams the log. From a shell, + `hermes computer-use screen status` prints the exact line and + `hermes computer-use screen install` runs it: + + | Distro | Packages | + |---|---| + | Debian / Ubuntu | `tigervnc-standalone-server xfce4-panel xfwm4 xfdesktop4 xfce4-settings xfce4-terminal dbus-x11 x11-xserver-utils x11-utils xauth fonts-dejavu-core` | + | Fedora | `tigervnc-server-minimal xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal dbus-x11 xsetroot xset xdpyinfo xprop xorg-x11-xauth setxkbmap dejavu-sans-fonts` | + | Arch | `tigervnc xfce4-panel xfwm4 xfdesktop xfce4-settings xfce4-terminal xorg-xsetroot xorg-xset xorg-xdpyinfo xorg-xprop xorg-xauth xorg-setxkbmap ttf-dejavu` | + + Deliberately **not** the `xfce4` metapackage: it pulls in the screensaver, + power manager and polkit agent that lock or prompt a headless desktop. +- [Computer Use](./computer-use.md) enabled for the bot (cua-driver installed). + +## Using it + +Every bot's computer is one click away in three places of Hermes Desktop: + +- **Bots → a bot → Scheduled Jobs**: the bot's screen is the hero at the very + top of the pane, above the title and the routines: a live preview of the + desktop (refreshed every few seconds while the pane is visible) with who holds + control; click the picture to expand into live access. While the screen is off + or not installed the same box says so and offers Start / Install. +- **Bots → right-click a bot → Open Screen**. +- **Sessions sidebar**, grouped by gateway / profile: the same **Screen** box + sits under each profile's header, so a profile's machine is reachable from + its conversations too. + +1. Open the Screen with any of the entries above. + The first time, click **Start screen**. Set `bot_desktop.auto_start: true` if + you want a headless host to start the screen by itself on the bot's first + `computer_use` call (off by default: installing TigerVNC never yields a screen + nobody asked for). A headed browser opens on the screen once it is running. +2. The pane streams the bot's desktop. The chip in the header says who is in + control: **Bot is in control** by default. +3. Click **Take over**. The border turns red, your keyboard and mouse now drive + the bot's screen. Sign in, solve the CAPTCHA, approve the payment. +4. Click **Hand back**. The bot regains control and re-captures the screen + before continuing. Closing the pane also hands control back. A *dropped* + connection is different: if your laptop lid closes or Wi-Fi drops while you + hold control, you keep it — the bot stays locked out of a screen you may be + mid-login on — until you reconnect and hand back. If you come back after a + reload and the pane still says a human holds control, a **Hand back (force)** + button appears to clear it. + +While you hold control, the bot's `computer_use` and browser tools are refused +with `human_has_control`, captures included. This is a tool-level fence, not an +OS one: the bot runs as the same user as its screen. Don't type secrets into a +bot you wouldn't trust with them. + +The bot can ask for you: when it recognises a login or verification step it +calls `computer_use` with `action: "request_handoff"` and a reason, the pane +shows **Bot needs you**, the bot tells you in its reply what it needs (so the +ask reaches you in whatever chat you are on), and it blocks in +`action: "wait_for_human"` until you hand back. + +Two viewers on one screen: the most recent **Take over** wins; the previous +controller drops back to watching. + +## Browser sessions that survive the handoff + +While the screen runs, the bot's browser tool and the dock's **Browser** icon are +the same browser: the Chromium agent-browser drives, with one persistent +user-data-dir per bot (`/bot-desktop/browser-profile`; set +`AGENT_BROWSER_PROFILE` to pin your own). Click Browser during a takeover and you +are in the bot's own windows and cookie jar; what you sign in to is what the bot +uses afterwards and in every later session, until the site expires the login. +Set `browser.headed: true` so the bot's own browsing is visible on the screen too. + +## CLI + +```bash +hermes computer-use screen status # installed? running? who holds control? +hermes computer-use screen start # start this profile's screen +hermes computer-use screen stop # stop it (hands control back first) +hermes computer-use screen install [-y] # apt/dnf/pacman the packages +hermes -p research computer-use screen start # another bot's screen +``` + +## Configuration + +```yaml +bot_desktop: + geometry: "1440x900" # screen size; the viewer scales to fit the pane + auto_start: true # start on the first computer_use call when the host has no display +``` + +State lives under `/bot-desktop/` per profile (RFB Unix socket, +Xauthority, launcher log, per-profile xfconf). + +## How it works + +- **TigerVNC `Xvnc`** is the X server and the RFB server in one process, per + profile, listening only on a `0600` Unix socket. No TCP port, no VNC + password: only processes running as the gateway's user can reach it (see the + threat model above), and the gateway's WebSocket bridge is the authenticated + way in. +- **Xfce** starts component-wise (`xfsettingsd`, `xfwm4 --compositor=off`, + `xfdesktop`, `xfce4-panel`) under a private D-Bus session, without + `xfce4-session`, so nothing tries to lock the screen or reach `logind`. +- **Hermes Desktop** bundles noVNC. It asks the gateway for a single-use ticket + (`display.observe`) over its normal authenticated connection and opens a + sibling WebSocket to `/api/display/ws`; the gateway splices the RFB stream + through. Nothing new is exposed; the pane works over local, SSH, URL+token + and Hermes Cloud connections alike. +- **Control lease.** The gateway drops keyboard, pointer and clipboard messages + from any viewer that does not hold the lease, at the RFB byte level; noVNC's + view-only flag is only the UI hint. The same lease gates `computer_use` and + the browser tools. It is a file under `/bot-desktop/`: no file + means the bot holds control (a fresh profile); a file that exists but cannot + be read or parsed fails closed — the bot is treated as locked out until the + next successful hand-off rewrites it. Xvnc never pushes the screen's clipboard + to viewers (`-SendCutText=0`), so watchers do not receive what the person in + control copies; pasting into the screen still works. +- **Display binding.** The launcher publishes `DISPLAY`, `XAUTHORITY` and the + D-Bus address; every cua-driver and headed-browser spawn for that profile + inherits them, so the bot never acts on a display a human is sitting at. + +## Troubleshooting + +- **"Screen packages missing"** — click **Install on host** in the pane, or run + the printed install line on the gateway host (not on the machine running + Hermes Desktop). The pane refuses a second install while one is running. +- **Screen starts then stops** — read `/bot-desktop/launcher.log`. +- **Typing produces wrong characters** — the screen uses a US keymap so RFB + keysyms and cua-driver agree; change it with `setxkbmap` on that `DISPLAY` + if you need another layout. +- **Bot says `human_has_control` after you left** — click **Hand back** in the + pane (or **Hand back (force)** after a reload). From a shell, + `hermes computer-use screen stop` releases the lease and stops the screen; + `hermes computer-use screen start` brings it back with the bot in control. diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md index 1148e2949a008..3bb53d2d20958 100644 --- a/website/docs/user-guide/features/computer-use.md +++ b/website/docs/user-guide/features/computer-use.md @@ -410,10 +410,11 @@ of screenshot context, not ~600K. [windows-ssh](https://cua.ai/docs/how-to-guides/driver/windows-ssh) has the recipe. - **Linux** requires a reachable display server. Headless servers - need Xvfb (`Xvfb :99 -screen 0 1920x1080x24`) before - `computer_use` can capture or inject events. Pure Wayland sessions - need an XWayland bridge for screen capture (cua-driver's Wayland - inject path handles input independently). + get one from [Bot Screen](./bot-screen.md): a per-profile Xfce + desktop over TigerVNC that Hermes starts on first use and streams + into Hermes Desktop, where you can take over for logins and 2FA. + Pure Wayland sessions need an XWayland bridge for screen capture + (cua-driver's Wayland inject path handles input independently). For cross-platform GUI automation without the desktop overhead (and without TCC / Session 0 / X11 setup), the `browser` toolset uses a diff --git a/website/sidebars.ts b/website/sidebars.ts index 55d1ed80160a0..621205071ac50 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -121,6 +121,7 @@ const sidebars: SidebarsConfig = { 'user-guide/features/browser', 'user-guide/features/credential-vault', 'user-guide/features/computer-use', + 'user-guide/features/bot-screen', 'user-guide/features/vision', 'user-guide/features/image-generation', 'user-guide/features/spotify',