From a3d3683695994a402a9c75dabc7dd1f6106bd7cb Mon Sep 17 00:00:00 2001 From: gus <42593099+lightcloud00@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:09:36 -0400 Subject: [PATCH 001/209] feat(local-vm): add secure two-up workspace --- electron/desktop-workspace.cjs | 274 ++++++++ electron/desktop-workspace.node-test.mjs | 241 ++++++++ electron/main.mjs | 83 ++- electron/preload.cjs | 12 + package.json | 2 +- server/chief-of-staff.test.ts | 10 + server/chief-of-staff.ts | 4 +- server/computer-control.test.ts | 45 ++ server/computer-control.ts | 55 +- server/drivers/acp/hermes.test.ts | 36 ++ server/drivers/acp/hermes.ts | 20 + server/index.test.ts | 40 ++ server/index.ts | 29 +- server/openmaus-status-capsule.test.ts | 301 +++++++++ server/openmaus-status-capsule.ts | 379 ++++++++++++ src/App.tsx | 45 +- src/components/CommandPalette.tsx | 6 +- src/components/ComputerPanel.tsx | 24 +- src/components/LocalVmWorkspace.tsx | 756 +++++++++++++++++++++++ src/lib/local-vm-workspace.test.ts | 198 ++++++ src/lib/local-vm-workspace.ts | 217 +++++++ src/types/ogb.d.ts | 33 + 22 files changed, 2801 insertions(+), 9 deletions(-) create mode 100644 electron/desktop-workspace.cjs create mode 100644 electron/desktop-workspace.node-test.mjs create mode 100644 server/drivers/acp/hermes.test.ts create mode 100644 server/openmaus-status-capsule.test.ts create mode 100644 server/openmaus-status-capsule.ts create mode 100644 src/components/LocalVmWorkspace.tsx create mode 100644 src/lib/local-vm-workspace.test.ts create mode 100644 src/lib/local-vm-workspace.ts diff --git a/electron/desktop-workspace.cjs b/electron/desktop-workspace.cjs new file mode 100644 index 000000000..419ea03a8 --- /dev/null +++ b/electron/desktop-workspace.cjs @@ -0,0 +1,274 @@ +const { desktopViewerUrl, sameDesktopViewerOrigin } = require("./desktop-viewer.cjs"); + +const MAX_WORKSPACE_VIEWS = 2; +const CONTEXT_ID = /^[A-Za-z0-9:_-]{1,120}$/; + +function isLoopbackHostname(hostname) { + return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "[::1]"; +} + +/** + * Local VM viewers are stricter than the existing cloud viewer: their noVNC + * endpoint must remain on this host. The view_only flag lives in noVNC's hash + * parameters alongside its short-lived password, so preserve every other + * field and change only that capability bit. + */ +function desktopWorkspaceUrl(rawUrl, interactive = false) { + const url = desktopViewerUrl(rawUrl); + if (!isLoopbackHostname(url.hostname)) { + throw new Error("Local VM desktops must use a loopback address"); + } + const fragment = new URLSearchParams(url.hash.slice(1)); + fragment.set("view_only", interactive ? "false" : "true"); + url.hash = fragment.toString(); + return url; +} + +function desktopWorkspaceIdentity(url) { + // Ports distinguish per-bot loopback viewers. Query/hash fields can contain + // credentials, so neither those fields nor a derivative of them is kept. + return `${url.protocol}//${url.host}${url.pathname}`; +} + +function desktopWorkspaceContextId(value) { + if (Object.prototype.toString.call(value) !== "[object String]" || !CONTEXT_ID.test(value)) { + throw new Error("The desktop workspace context is invalid"); + } + return value; +} + +function normalizeDesktopWorkspaceBounds(rawBounds, contentSize) { + if (Object.prototype.toString.call(rawBounds) !== "[object Object]") { + throw new Error("Desktop workspace bounds are invalid"); + } + if (!Array.isArray(contentSize) || contentSize.length !== 2) { + throw new Error("The desktop workspace owner size is unavailable"); + } + const values = [rawBounds.x, rawBounds.y, rawBounds.width, rawBounds.height]; + if (values.some((value) => !Number.isFinite(value))) { + throw new Error("Desktop workspace bounds are invalid"); + } + + const ownerWidth = Math.max(1, Math.floor(contentSize[0])); + const ownerHeight = Math.max(1, Math.floor(contentSize[1])); + let x = Math.round(rawBounds.x); + let y = Math.round(rawBounds.y); + let width = Math.round(rawBounds.width); + let height = Math.round(rawBounds.height); + if (width < 1 || height < 1) throw new Error("Desktop workspace bounds are empty"); + + x = Math.max(0, Math.min(x, ownerWidth - 1)); + y = Math.max(0, Math.min(y, ownerHeight - 1)); + width = Math.max(1, Math.min(width, ownerWidth - x)); + height = Math.max(1, Math.min(height, ownerHeight - y)); + return { x, y, width, height }; +} + +function createDesktopWorkspaceManager({ owner, createView, notify, partitionPrefix }) { + if (!owner || owner.isDestroyed?.()) throw new Error("The OpenMausBot window is unavailable"); + if (createView?.constructor !== Function) throw new Error("The desktop workspace viewer is unavailable"); + const emit = notify?.constructor === Function ? notify : () => {}; + const entries = new Map(); + let partitionCounter = 0; + let interactiveOperation = Promise.resolve(); + + const serializeInteractiveChange = (operation) => { + const pending = interactiveOperation.catch(() => {}).then(operation); + // A failed reload must fail its caller without poisoning later demotions. + interactiveOperation = pending.catch(() => {}); + return pending; + }; + + const stateFor = (entry, status, code) => { + const state = { + contextId: entry.contextId, + open: status !== "closed", + status, + interactive: entry.interactive, + }; + if (code) state.code = code; + return state; + }; + + const removeEntry = (entry, status = "closed", code) => { + if (entries.get(entry.contextId) !== entry) return; + entries.delete(entry.contextId); + try { + entry.view.setVisible(false); + } catch {} + try { + owner.contentView.removeChildView(entry.view); + } catch {} + try { + if (!entry.view.webContents.isDestroyed()) { + entry.view.webContents.close({ waitForBeforeUnload: false }); + } + } catch {} + emit(stateFor(entry, status, code)); + }; + + const secureView = (entry, viewerOrigin) => { + const contents = entry.view.webContents; + contents.session.setPermissionCheckHandler(() => false); + contents.session.setPermissionRequestHandler((_webContents, _permission, callback) => callback(false)); + contents.setWindowOpenHandler(() => ({ action: "deny" })); + + const keepOnOrigin = (event, target) => { + if (sameDesktopViewerOrigin(target, viewerOrigin)) return; + event.preventDefault(); + }; + contents.on("will-navigate", keepOnOrigin); + contents.on("will-redirect", keepOnOrigin); + contents.on("did-fail-load", (_event, code, _description, _failedUrl, isMainFrame) => { + if (!isMainFrame || code === -3 || entries.get(entry.contextId) !== entry) return; + removeEntry(entry, "error", "load-failed"); + }); + contents.on("render-process-gone", () => { + if (entries.get(entry.contextId) === entry) removeEntry(entry, "error", "renderer-gone"); + }); + }; + + const loadMode = async (entry, interactive) => { + const current = entry.view.webContents.getURL(); + const next = desktopWorkspaceUrl(current, interactive); + entry.interactive = interactive; + emit(stateFor(entry, "opening")); + try { + await entry.view.webContents.loadURL(next.toString()); + } catch { + // A failed demotion must never leave an old interactive noVNC document + // receiving input. Remove the native view entirely and fail closed. + removeEntry(entry, "error", "load-failed"); + throw new Error("The Local VM desktop did not load"); + } + if (entries.get(entry.contextId) === entry) emit(stateFor(entry, "ready")); + }; + + return { + async open(input) { + if (Object.prototype.toString.call(input) !== "[object Object]") { + throw new Error("Desktop workspace input is invalid"); + } + const contextId = desktopWorkspaceContextId(input.contextId); + if (entries.has(contextId)) throw new Error("That desktop workspace slot is already open"); + if (entries.size >= MAX_WORKSPACE_VIEWS) { + throw new Error("Only two Local VM desktops can be open together"); + } + + const url = desktopWorkspaceUrl(input.url, false); + const identity = desktopWorkspaceIdentity(url); + if ([...entries.values()].some((entry) => entry.identity === identity)) { + throw new Error("That Local VM desktop is already open"); + } + const bounds = normalizeDesktopWorkspaceBounds(input.bounds, owner.getContentSize()); + const partition = `${partitionPrefix}-${++partitionCounter}`; + const view = createView({ + webPreferences: { + nodeIntegration: false, + contextIsolation: true, + sandbox: true, + webSecurity: true, + allowRunningInsecureContent: false, + // No persist: prefix: each pane receives a private in-memory session. + partition, + }, + }); + const entry = { contextId, view, identity, interactive: false }; + entries.set(contextId, entry); + secureView(entry, url.origin); + view.setBounds(bounds); + // The renderer explicitly lays the view out after the DOM rectangle is + // stable. Keeping it hidden here also prevents a native view from + // flashing above a modal during setup. + view.setVisible(false); + owner.contentView.addChildView(view); + emit(stateFor(entry, "opening")); + try { + await view.webContents.loadURL(url.toString()); + } catch { + removeEntry(entry, "error", "load-failed"); + throw new Error("The Local VM desktop did not load"); + } + if (entries.get(contextId) === entry) emit(stateFor(entry, "ready")); + return stateFor(entry, "ready"); + }, + + layout(items) { + if (!Array.isArray(items) || items.length > MAX_WORKSPACE_VIEWS) { + throw new Error("Desktop workspace layout is invalid"); + } + const seen = new Set(); + for (const item of items) { + if (Object.prototype.toString.call(item) !== "[object Object]") { + throw new Error("Desktop workspace layout is invalid"); + } + const contextId = desktopWorkspaceContextId(item.contextId); + if (seen.has(contextId)) throw new Error("Desktop workspace layout contains a duplicate slot"); + seen.add(contextId); + const entry = entries.get(contextId); + if (!entry) throw new Error("That desktop workspace slot is not open"); + const bounds = normalizeDesktopWorkspaceBounds(item.bounds, owner.getContentSize()); + entry.view.setBounds(bounds); + entry.view.setVisible(item.visible === true); + } + return true; + }, + + setInteractive(rawContextId) { + const contextId = rawContextId == null ? null : desktopWorkspaceContextId(rawContextId); + const scopedEntries = [...entries.values()]; + const targetEntry = contextId === null ? null : entries.get(contextId); + if (contextId !== null && !targetEntry) { + return Promise.reject(new Error("That desktop workspace slot is not open")); + } + return serializeInteractiveChange(async () => { + if (targetEntry && entries.get(contextId) !== targetEntry) { + throw new Error("That desktop workspace slot is not open"); + } + // Always finish every demotion before promoting. The queue is part of + // this invariant: overlapping renderer IPC calls cannot observe a flag + // change while the old interactive noVNC document is still reloading. + for (const entry of scopedEntries) { + if ( + entries.get(entry.contextId) === entry && + entry.interactive && + entry.contextId !== contextId + ) { + await loadMode(entry, false); + } + } + if (targetEntry && !targetEntry.interactive) { + await loadMode(targetEntry, true); + } + return true; + }); + }, + + close(rawContextId) { + if (rawContextId == null) { + for (const entry of entries.values()) removeEntry(entry); + return true; + } + const contextId = desktopWorkspaceContextId(rawContextId); + const entry = entries.get(contextId); + if (entry) removeEntry(entry); + return true; + }, + + closeAll() { + for (const entry of entries.values()) removeEntry(entry); + }, + + size() { + return entries.size; + }, + }; +} + +module.exports = { + MAX_WORKSPACE_VIEWS, + createDesktopWorkspaceManager, + desktopWorkspaceContextId, + desktopWorkspaceUrl, + normalizeDesktopWorkspaceBounds, +}; diff --git a/electron/desktop-workspace.node-test.mjs b/electron/desktop-workspace.node-test.mjs new file mode 100644 index 000000000..5a71d412c --- /dev/null +++ b/electron/desktop-workspace.node-test.mjs @@ -0,0 +1,241 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import test from "node:test"; + +const require = createRequire(import.meta.url); +const { + createDesktopWorkspaceManager, + desktopWorkspaceUrl, + normalizeDesktopWorkspaceBounds, +} = require("./desktop-workspace.cjs"); + +test("workspace URLs stay loopback and force the requested noVNC input mode", () => { + const watch = desktopWorkspaceUrl( + "http://127.0.0.1:6080/vnc.html#autoconnect=true&resize=scale&password=secret123", + ); + assert.equal(watch.hostname, "127.0.0.1"); + assert.equal(watch.hash.includes("autoconnect=true"), true); + assert.equal(watch.hash.includes("resize=scale"), true); + assert.equal(watch.hash.includes("password=secret123"), true); + assert.equal(watch.hash.includes("view_only=true"), true); + + const interactive = desktopWorkspaceUrl(watch.toString(), true); + assert.equal(interactive.hash.includes("view_only=false"), true); + assert.equal(interactive.hash.includes("view_only=true"), false); + assert.doesNotThrow(() => desktopWorkspaceUrl("https://localhost:6080/vnc.html")); + assert.doesNotThrow(() => desktopWorkspaceUrl("http://[::1]:6080/vnc.html")); + assert.throws(() => desktopWorkspaceUrl("https://desktop.example/vnc.html"), /loopback/); +}); + +test("workspace URL errors never echo a secret-bearing input", () => { + const secret = "never-print-this"; + assert.throws( + () => desktopWorkspaceUrl(`https://desktop.example/vnc.html#password=${secret}`), + (error) => error instanceof Error && !error.message.includes(secret), + ); +}); + +test("workspace bounds reject malformed values and clamp to owner content", () => { + assert.deepEqual( + normalizeDesktopWorkspaceBounds({ x: 901, y: -5, width: 500, height: 900 }, [1000, 800]), + { x: 901, y: 0, width: 99, height: 800 }, + ); + assert.throws( + () => normalizeDesktopWorkspaceBounds({ x: 0, y: 0, width: "20", height: 20 }, [1000, 800]), + /invalid/, + ); + assert.throws( + () => normalizeDesktopWorkspaceBounds({ x: 0, y: 0, width: 0, height: 20 }, [1000, 800]), + /empty/, + ); +}); + +function managerFixture() { + const notifications = []; + const views = []; + const children = []; + class FakeWebContents { + constructor() { + this.url = ""; + this.closed = false; + this.handlers = new Map(); + this.session = { + setPermissionCheckHandler: (handler) => { this.permissionCheck = handler; }, + setPermissionRequestHandler: (handler) => { this.permissionRequest = handler; }, + }; + } + setWindowOpenHandler(handler) { this.windowOpenHandler = handler; } + on(name, handler) { this.handlers.set(name, handler); } + async loadURL(url) { + if (this.loadHook) await this.loadHook(url); + this.url = url; + } + getURL() { return this.url; } + isDestroyed() { return this.closed; } + close() { this.closed = true; } + } + class FakeView { + constructor(options) { + this.options = options; + this.webContents = new FakeWebContents(); + this.visible = false; + this.bounds = null; + views.push(this); + } + setBounds(bounds) { this.bounds = bounds; } + setVisible(visible) { this.visible = visible; } + } + const owner = { + contentView: { + addChildView(view) { children.push(view); }, + removeChildView(view) { + const index = children.indexOf(view); + if (index >= 0) children.splice(index, 1); + }, + }, + getContentSize: () => [1200, 800], + isDestroyed: () => false, + }; + const manager = createDesktopWorkspaceManager({ + owner, + createView: (options) => new FakeView(options), + notify: (state) => notifications.push(state), + partitionPrefix: "openmausbot-test", + }); + const open = (contextId, port, bounds = { x: 10, y: 20, width: 500, height: 400 }) => + manager.open({ + contextId, + url: `http://127.0.0.1:${port}/vnc.html#autoconnect=true&password=secret-${port}`, + title: contextId, + bounds, + }); + return { children, manager, notifications, open, views }; +} + +test("manager keeps two isolated watch-only views and rejects duplicates or a third", async () => { + const { children, manager, open, views } = managerFixture(); + await open("left", 6080); + await open("right", 6081); + assert.equal(manager.size(), 2); + assert.equal(children.length, 2); + assert.notEqual( + views[0].options.webPreferences.partition, + views[1].options.webPreferences.partition, + ); + assert.equal(views.every((view) => view.webContents.url.includes("view_only=true")), true); + assert.equal(views.every((view) => view.options.webPreferences.sandbox === true), true); + assert.equal(views.every((view) => view.webContents.permissionCheck() === false), true); + assert.equal(views.every((view) => view.webContents.windowOpenHandler().action === "deny"), true); + assert.equal( + views.every((view) => !view.options.webPreferences.partition.startsWith("persist:")), + true, + ); + let denied = null; + views[0].webContents.permissionRequest(null, "camera", (allowed) => { denied = allowed; }); + assert.equal(denied, false); + let prevented = false; + views[0].webContents.handlers.get("will-navigate")( + { preventDefault() { prevented = true; } }, + "https://example.com/steal", + ); + assert.equal(prevented, true); + prevented = false; + views[0].webContents.handlers.get("will-navigate")( + { preventDefault() { prevented = true; } }, + "http://127.0.0.1:6080/another-local-path", + ); + assert.equal(prevented, false); + await assert.rejects(() => open("left", 6082), /already open/); + await assert.rejects(() => open("third", 6082), /Only two/); + + manager.close("right"); + await assert.rejects(() => open("third", 6080), /already open/); +}); + +test("manager lays out panes and demotes the old pane before promoting the new one", async () => { + const { manager, open, views } = managerFixture(); + await open("left", 6080); + await open("right", 6081); + manager.layout([ + { contextId: "left", bounds: { x: 20, y: 60, width: 550, height: 600 }, visible: true }, + { contextId: "right", bounds: { x: 590, y: 60, width: 550, height: 600 }, visible: true }, + ]); + assert.equal(views[0].visible, true); + assert.deepEqual(views[1].bounds, { x: 590, y: 60, width: 550, height: 600 }); + + await manager.setInteractive("left"); + assert.equal(views[0].webContents.url.includes("view_only=false"), true); + assert.equal(views[1].webContents.url.includes("view_only=true"), true); + await manager.setInteractive("right"); + assert.equal(views[0].webContents.url.includes("view_only=true"), true); + assert.equal(views[1].webContents.url.includes("view_only=false"), true); + await manager.setInteractive(null); + assert.equal(views.every((view) => view.webContents.url.includes("view_only=true")), true); +}); + +test("manager serializes overlapping demotion and promotion calls", async () => { + const { manager, open, views } = managerFixture(); + await open("left", 6080); + await open("right", 6081); + await manager.setInteractive("left"); + + let finishDemotion; + const demotionGate = new Promise((resolve) => { finishDemotion = resolve; }); + let rightPromotionStarted = false; + views[0].webContents.loadHook = async (url) => { + if (url.includes("view_only=true")) await demotionGate; + }; + views[1].webContents.loadHook = async (url) => { + if (url.includes("view_only=false")) rightPromotionStarted = true; + }; + + const demote = manager.setInteractive(null); + await new Promise((resolve) => setImmediate(resolve)); + const promote = manager.setInteractive("right"); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(rightPromotionStarted, false); + assert.equal(views[0].webContents.url.includes("view_only=false"), true); + + finishDemotion(); + await Promise.all([demote, promote]); + assert.equal(views[0].webContents.url.includes("view_only=true"), true); + assert.equal(views[1].webContents.url.includes("view_only=false"), true); +}); + +test("queued interaction cannot promote a replacement pane with a reused context id", async () => { + const { manager, open, views } = managerFixture(); + await open("left", 6080); + await open("right", 6081); + await manager.setInteractive("left"); + + let finishDemotion; + const demotionGate = new Promise((resolve) => { finishDemotion = resolve; }); + views[0].webContents.loadHook = async (url) => { + if (url.includes("view_only=true")) await demotionGate; + }; + + const demote = manager.setInteractive(null); + await new Promise((resolve) => setImmediate(resolve)); + const stalePromotion = manager.setInteractive("right"); + manager.close("right"); + await open("right", 6082); + + finishDemotion(); + await demote; + await assert.rejects(stalePromotion, /not open/); + assert.equal(views[2].webContents.url.includes("view_only=true"), true); +}); + +test("manager closes panes independently and emits no viewer URL", async () => { + const { children, manager, notifications, open, views } = managerFixture(); + await open("left", 6080); + await open("right", 6081); + manager.close("left"); + assert.equal(children.length, 1); + assert.equal(views[0].webContents.closed, true); + assert.equal(views[1].webContents.closed, false); + manager.closeAll(); + assert.equal(children.length, 0); + assert.equal(JSON.stringify(notifications).includes("password="), false); + assert.equal(JSON.stringify(notifications).includes("127.0.0.1"), false); +}); diff --git a/electron/main.mjs b/electron/main.mjs index b7d0dd5f7..cd2ffed6f 100644 --- a/electron/main.mjs +++ b/electron/main.mjs @@ -1,4 +1,5 @@ -import { app, BrowserWindow, clipboard, desktopCapturer, dialog, ipcMain, safeStorage, screen, session, shell, systemPreferences, utilityProcess } from "electron"; +import { app, BrowserWindow, WebContentsView, clipboard, desktopCapturer, dialog, ipcMain, safeStorage, screen, session, shell, systemPreferences, utilityProcess } from "electron"; +import { randomUUID } from "node:crypto"; import { createRequire } from "node:module"; import fs from "node:fs"; import path from "node:path"; @@ -19,6 +20,7 @@ const { createDisplayMediaGuard, invokeDisplayMediaCallback, selectCaptureSource ); const { STAGE_PREFIX: APPIMAGE_CUA_STAGE_PREFIX } = require("./cua-linux-bundle.cjs"); const { desktopViewerUrl, sameDesktopViewerOrigin } = require("./desktop-viewer.cjs"); +const { createDesktopWorkspaceManager } = require("./desktop-workspace.cjs"); const __dirname = path.dirname(fileURLToPath(import.meta.url)); // 127.0.0.1 explicitly — vite binds IPv4; a bare "localhost" here can @@ -30,6 +32,9 @@ const APP_ICON = path.join(__dirname, "resources/app-icon.png"); let desktopViewerWindow = null; let desktopViewerOwner = null; let desktopViewerContextId = null; +let desktopWorkspaceManager = null; +let desktopWorkspaceOwner = null; +let mainWindow = null; // GNOME groups the window with its installed desktop entry only when both // identities match. This must run before Electron becomes ready. @@ -423,6 +428,56 @@ function openDesktopViewer(owner, rawUrl, rawTitle, contextId) { return true; } +function ensureDesktopWorkspace(owner) { + if (!owner || owner.isDestroyed()) throw new Error("The OpenMausBot window is unavailable"); + if (desktopWorkspaceManager) { + if (desktopWorkspaceOwner !== owner) { + throw new Error("The desktop workspace belongs to another app window"); + } + return desktopWorkspaceManager; + } + + desktopWorkspaceOwner = owner; + const manager = createDesktopWorkspaceManager({ + owner, + createView: (options) => new WebContentsView(options), + partitionPrefix: `openmausbot-desktop-workspace-${randomUUID()}`, + notify: (state) => { + if (!owner.isDestroyed() && !owner.webContents.isDestroyed()) { + owner.webContents.send("desktop-workspace:state", state); + } + }, + }); + desktopWorkspaceManager = manager; + + // Native child views outlive the renderer DOM unless we explicitly tear + // them down. Reloads, renderer crashes and owner destruction all close both + // panes without retaining their secret-bearing noVNC URLs. + owner.webContents.on("did-start-navigation", (_event, _url, isInPlace, isMainFrame) => { + if (isMainFrame && !isInPlace) manager.closeAll(); + }); + owner.webContents.on("render-process-gone", () => manager.closeAll()); + owner.once("closed", () => { + manager.closeAll(); + if (desktopWorkspaceManager === manager) { + desktopWorkspaceManager = null; + desktopWorkspaceOwner = null; + } + }); + return manager; +} + +function desktopWorkspaceForEvent(event, create = false) { + const owner = mainWindow; + if (!owner || owner.isDestroyed() || event.sender !== owner.webContents) { + throw new Error("The desktop workspace is available only to the main app window"); + } + if (desktopWorkspaceManager && desktopWorkspaceOwner !== owner) { + throw new Error("The desktop workspace belongs to another app window"); + } + return create ? ensureDesktopWorkspace(owner) : desktopWorkspaceManager; +} + ipcMain.on("screen:preview-intent", (event) => { event.returnValue = displayMediaGuard.begin(event.senderFrame); }); @@ -456,6 +511,10 @@ function createWindow() { preload: path.join(__dirname, "preload.cjs"), }, }); + mainWindow = win; + win.once("closed", () => { + if (mainWindow === win) mainWindow = null; + }); win.webContents.setWindowOpenHandler(({ url }) => { shell.openExternal(url); @@ -645,6 +704,28 @@ ipcMain.handle("desktop-viewer:open", (event, rawUrl, title, contextId) => { return openDesktopViewer(owner, rawUrl, title, contextId); }); +// Two Local VM desktops share the existing app BrowserWindow. The renderer +// supplies only layout and intent; URL validation, sandboxing, session +// isolation and the one-interactive-pane invariant stay in the main process. +ipcMain.handle("desktop-workspace:open", (event, input) => + desktopWorkspaceForEvent(event, true).open(input), +); +ipcMain.handle("desktop-workspace:layout", (event, items) => { + const manager = desktopWorkspaceForEvent(event); + if (!manager) return false; + return manager.layout(items); +}); +ipcMain.handle("desktop-workspace:set-interactive", (event, contextId) => { + const manager = desktopWorkspaceForEvent(event); + if (!manager) return contextId == null; + return manager.setInteractive(contextId); +}); +ipcMain.handle("desktop-workspace:close", (event, contextId) => { + const manager = desktopWorkspaceForEvent(event); + if (!manager) return true; + return manager.close(contextId); +}); + ipcMain.handle("perm:status", () => ({ mic: nativeActions.appleMediaPermissions diff --git a/electron/preload.cjs b/electron/preload.cjs index 14f2646f9..cee8b3fd6 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -86,6 +86,18 @@ contextBridge.exposeInMainWorld("ogb", { return () => ipcRenderer.removeListener("desktop-viewer:state", handler); }, }, + /** Two sandboxed Local VM viewers embedded in the owning app window. */ + desktopWorkspace: { + open: (input) => ipcRenderer.invoke("desktop-workspace:open", input), + layout: (items) => ipcRenderer.invoke("desktop-workspace:layout", items), + setInteractive: (contextId) => ipcRenderer.invoke("desktop-workspace:set-interactive", contextId), + close: (contextId) => ipcRenderer.invoke("desktop-workspace:close", contextId), + onState: (cb) => { + const handler = (_event, state) => cb(state); + ipcRenderer.on("desktop-workspace:state", handler); + return () => ipcRenderer.removeListener("desktop-workspace:state", handler); + }, + }, /** Native folder picker for a bot's working folder; null when cancelled. */ pickFolder: (current) => ipcRenderer.invoke("desktop:pick-folder", current), /** Store a provider credential with OS-backed encryption. */ diff --git a/package.json b/package.json index 29d66525e..c594d4d36 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typecheck": "tsc -b && tsc -p tsconfig.server.json", "test": "node scripts/test-floor.mjs && pnpm broker:test && pnpm test:updater && pnpm test:desktop-viewer && pnpm test:packaged-server", "test:updater": "node --test electron/updater-coordinator.node-test.mjs", - "test:desktop-viewer": "node --test electron/desktop-viewer.node-test.mjs", + "test:desktop-viewer": "node --test electron/desktop-viewer.node-test.mjs electron/desktop-workspace.node-test.mjs", "bench:observation": "node --experimental-strip-types scripts/bench-observation.ts", "test:watch": "vitest", "test:cua": "pnpm build:cua && node scripts/smoke-cua.mjs", diff --git a/server/chief-of-staff.test.ts b/server/chief-of-staff.test.ts index df47c8623..0663d918b 100644 --- a/server/chief-of-staff.test.ts +++ b/server/chief-of-staff.test.ts @@ -58,4 +58,14 @@ describe("chiefOfStaffSystemPrompt", () => { expect(prompt).toContain("cannot contact teammates"); expect(prompt).not.toContain("Use ask_bot"); }); + + it("includes trusted OpenMaus status only when the Chief caller supplies it", () => { + const status = "TRUSTED OPENMAUSBOT STATUS\nfreshness=fresh; runtime_state=degraded"; + + const chiefPrompt = chiefOfStaffSystemPrompt("chief", bots, true, status); + const ordinaryPrompt = chiefOfStaffSystemPrompt("writer", bots, true); + + expect(chiefPrompt).toContain(status); + expect(ordinaryPrompt).not.toContain("TRUSTED OPENMAUSBOT STATUS"); + }); }); diff --git a/server/chief-of-staff.ts b/server/chief-of-staff.ts index 5327c2994..1dfe67d42 100644 --- a/server/chief-of-staff.ts +++ b/server/chief-of-staff.ts @@ -27,6 +27,7 @@ export function chiefOfStaffSystemPrompt( chiefId: string, bots: ChiefTeamMember[], canDelegate: boolean, + trustedOpenMausStatus = "", ): string { const team = bots.filter((bot) => bot.id !== chiefId && !bot.hidden); const listed = team.slice(0, ROSTER_MAX_BOTS); @@ -58,5 +59,6 @@ export function chiefOfStaffSystemPrompt( delegation, "Current workspace team:", roster, - ].join("\n"); + trustedOpenMausStatus, + ].filter(Boolean).join("\n"); } diff --git a/server/computer-control.test.ts b/server/computer-control.test.ts index ac22cb069..8e5d12020 100644 --- a/server/computer-control.test.ts +++ b/server/computer-control.test.ts @@ -35,6 +35,51 @@ describe("computer control", () => { expect(control.take("b1").heldSinceMs).toBe(1000); }); + it("atomically acquires a workspace lease without exposing its id", () => { + const { control, changes } = tracked(); + const leaseId = "5b6bbbd2-b88b-4c50-a748-ec87f332662f"; + const acquired = control.acquireLease("b1", leaseId); + expect(acquired).toMatchObject({ owned: true, acquired: true, snapshot: { held: true } }); + expect(acquired.snapshot).not.toHaveProperty("controlLeaseId"); + expect(JSON.stringify(changes)).not.toContain(leaseId); + + const sameLease = control.acquireLease("b1", leaseId); + expect(sameLease).toMatchObject({ owned: true, acquired: false }); + expect(changes).toHaveLength(1); + }); + + it("does not acquire or release a hold owned by another surface", () => { + const { control, changes } = tracked(); + control.take("b1"); + const leaseId = "57c7f3ef-e41d-4adf-bbda-0bd25bb03893"; + + expect(control.acquireLease("b1", leaseId)).toMatchObject({ + owned: false, + acquired: false, + snapshot: { held: true }, + }); + expect(control.releaseLease("b1", leaseId)).toMatchObject({ + released: false, + snapshot: { held: true }, + }); + expect(changes.map((change) => change.snapshot.held)).toEqual([true]); + }); + + it("conditionally releases only the matching workspace lease", () => { + const { control, changes } = tracked(); + const owner = "33e62f3a-89d9-4117-b48a-15f7deae3252"; + const other = "ed602995-306f-480a-8817-e8d8c8fe7d90"; + control.acquireLease("b1", owner); + + expect(control.releaseLease("b1", other).released).toBe(false); + expect(control.snapshot("b1").held).toBe(true); + expect(control.releaseLease("b1", owner)).toMatchObject({ + released: true, + snapshot: { held: false }, + }); + expect(changes.map((change) => change.snapshot.held)).toEqual([true, false]); + }); + it("requestHelp surfaces the plea but never grants control", () => { const { control } = tracked(); const snapshot = control.requestHelp("b1", " please log in for me "); diff --git a/server/computer-control.ts b/server/computer-control.ts index 604d6492c..18c9e663d 100644 --- a/server/computer-control.ts +++ b/server/computer-control.ts @@ -25,6 +25,20 @@ export interface ControlSnapshot { heldSinceMs: number | null; } +export interface ControlLeaseResult { + snapshot: ControlSnapshot; + /** True only when this lease currently owns the hold. */ + owned: boolean; + /** True only when this call changed an unheld record into a held one. */ + acquired: boolean; +} + +export interface ControlLeaseReleaseResult { + snapshot: ControlSnapshot; + /** True only when this call removed a hold owned by the supplied lease. */ + released: boolean; +} + const NO_CONTROL: ControlSnapshot = { held: false, helpReason: null, heldSinceMs: null }; /** Keep a shouted help reason card-sized; the transcript has the rest. */ const MAX_REASON_CHARS = 280; @@ -33,6 +47,8 @@ interface Entry { heldSinceMs: number | null; helpReason: string | null; helpRequestId: string | null; + /** Opaque workspace lease. It is deliberately absent from every snapshot. */ + controlLeaseId: string | null; } export class ComputerControl { @@ -68,10 +84,31 @@ export class ComputerControl { heldSinceMs: this.now(), helpReason: entry?.helpReason ?? null, helpRequestId: entry?.helpRequestId ?? null, + controlLeaseId: null, }); return this.changed(botId); } + /** Atomically take or re-check a workspace-owned hold. The opaque lease is + * never returned in a snapshot, broadcast, or API response. */ + acquireLease(botId: string, controlLeaseId: string): ControlLeaseResult { + const entry = this.entries.get(botId); + if (entry?.heldSinceMs != null) { + return { + snapshot: this.snapshot(botId), + owned: entry.controlLeaseId === controlLeaseId, + acquired: false, + }; + } + this.entries.set(botId, { + heldSinceMs: this.now(), + helpReason: entry?.helpReason ?? null, + helpRequestId: entry?.helpRequestId ?? null, + controlLeaseId, + }); + return { snapshot: this.changed(botId), owned: true, acquired: true }; + } + /** The person hands the wheel back. Also settles any open help request — * the waiting bot resumes from this one state change. */ release(botId: string): ControlSnapshot { @@ -80,6 +117,17 @@ export class ComputerControl { return this.changed(botId); } + /** Release only the hold created by this workspace lease. A newer or legacy + * holder is observed but never disturbed. */ + releaseLease(botId: string, controlLeaseId: string): ControlLeaseReleaseResult { + const entry = this.entries.get(botId); + if (!entry || entry.heldSinceMs === null || entry.controlLeaseId !== controlLeaseId) { + return { snapshot: this.snapshot(botId), released: false }; + } + this.entries.delete(botId); + return { snapshot: this.changed(botId), released: true }; + } + /** The bot asks the person to take over. Never grants anything by * itself — it only surfaces the plea. A reason shouted while the person * is already driving is kept, but must not clobber an earlier one they @@ -92,7 +140,12 @@ export class ComputerControl { * this id to expire only its own unanswered plea when its wait ends. */ requestHelpLease(botId: string, reason: unknown): { snapshot: ControlSnapshot; requestId: string } { const text = typeof reason === "string" ? reason.trim().slice(0, MAX_REASON_CHARS) : ""; - const entry = this.entries.get(botId) ?? { heldSinceMs: null, helpReason: null, helpRequestId: null }; + const entry = this.entries.get(botId) ?? { + heldSinceMs: null, + helpReason: null, + helpRequestId: null, + controlLeaseId: null, + }; if (entry.helpReason === null) { entry.helpReason = text || "the bot asked you to take over"; entry.helpRequestId = `${botId}-${++this.requestSequence}`; diff --git a/server/drivers/acp/hermes.test.ts b/server/drivers/acp/hermes.test.ts new file mode 100644 index 000000000..5e63094aa --- /dev/null +++ b/server/drivers/acp/hermes.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import { + HERMES_OPENMAUS_SCREENSHOT_COMPAT, + HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL, + bindHermesScreenshotCompat, +} from "./hermes.ts"; + +describe("Hermes OpenMaus screenshot compatibility binding", () => { + it("binds the exact leaf model for an injected local picker model", () => { + const env = { + [HERMES_OPENMAUS_SCREENSHOT_COMPAT]: undefined, + [HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL]: undefined, + }; + + bindHermesScreenshotCompat(env, "omlx::gemma-4-31b-it-bf16"); + + expect(env[HERMES_OPENMAUS_SCREENSHOT_COMPAT]).toBe("1"); + expect(env[HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL]).toBe("gemma-4-31b-it-bf16"); + }); + + it.each([undefined, "", "anthropic/claude-opus-4.6", "unknown::model"])( + "clears inherited compatibility for an unbound model %s", + (model) => { + const env = { + [HERMES_OPENMAUS_SCREENSHOT_COMPAT]: "1", + [HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL]: "stale/model", + }; + + bindHermesScreenshotCompat(env, model); + + expect(env[HERMES_OPENMAUS_SCREENSHOT_COMPAT]).toBeUndefined(); + expect(env[HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL]).toBeUndefined(); + }, + ); +}); diff --git a/server/drivers/acp/hermes.ts b/server/drivers/acp/hermes.ts index 09c4561f6..45537382c 100644 --- a/server/drivers/acp/hermes.ts +++ b/server/drivers/acp/hermes.ts @@ -14,6 +14,22 @@ import { createAcpDriver, type AcpSupport } from "./core.ts"; const EMPTY: ModelCatalog = { default: "", options: [] }; +export const HERMES_OPENMAUS_SCREENSHOT_COMPAT = "HERMES_OPENMAUS_SCREENSHOT_COMPAT"; +export const HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL = "HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL"; + +/** Bind screenshot pseudo-call compatibility to one exact injected model. */ +export function bindHermesScreenshotCompat( + env: Record, + modelId: string | null | undefined, +): void { + delete env[HERMES_OPENMAUS_SCREENSHOT_COMPAT]; + delete env[HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL]; + const inject = decodeInjectId(modelId); + if (!inject) return; + env[HERMES_OPENMAUS_SCREENSHOT_COMPAT] = "1"; + env[HERMES_OPENMAUS_SCREENSHOT_COMPAT_MODEL] = inject.model; +} + function hermesHome(env: Record): string { return env.HERMES_HOME || join(env.HOME || env.USERPROFILE || homedir(), ".hermes"); } @@ -107,6 +123,10 @@ const support: AcpSupport = { models: EMPTY, resolveModels, resolveTurnModel: (model, env) => { + // Never inherit a broad or stale compatibility grant from the parent. + // Only this OpenMaus driver binds one concrete local model; Hermes still + // requires the exact read-only screenshot MCP tool before activation. + bindHermesScreenshotCompat(env, model); if (!model) return model; ensureHermesInjectProvider(model, env); return model; diff --git a/server/index.test.ts b/server/index.test.ts index 3819f5b63..b8ada247c 100644 --- a/server/index.test.ts +++ b/server/index.test.ts @@ -1937,6 +1937,46 @@ describe("computer control API (who is driving)", () => { } }); + it("atomically owns and conditionally releases a workspace lease without returning its id", async () => { + const owner = "lease_5b6bbbd2-b88b-4c50-a748-ec87f332662f"; + const other = "lease_ed602995-306f-480a-8817-e8d8c8fe7d90"; + const took = await api("POST", `/api/bots/${botId}/computer/control`, { + action: "take", + controlLeaseId: owner, + }); + expect(took.body).toMatchObject({ held: true, owned: true, acquired: true }); + expect(JSON.stringify(took.body)).not.toContain(owner); + + const blocked = await api("POST", `/api/bots/${botId}/computer/control`, { + action: "take", + controlLeaseId: other, + }); + expect(blocked.body).toMatchObject({ held: true, owned: false, acquired: false }); + + const wrongRelease = await api("POST", `/api/bots/${botId}/computer/control`, { + action: "release", + controlLeaseId: other, + }); + expect(wrongRelease.body).toMatchObject({ held: true, released: false }); + + const released = await api("POST", `/api/bots/${botId}/computer/control`, { + action: "release", + controlLeaseId: owner, + }); + expect(released.body).toMatchObject({ held: false, released: true }); + expect(JSON.stringify(released.body)).not.toContain(owner); + }); + + it("rejects malformed workspace leases without echoing them", async () => { + const invalid = "bad lease value"; + const res = await api("POST", `/api/bots/${botId}/computer/control`, { + action: "take", + controlLeaseId: invalid, + }); + expect(res.status).toBe(400); + expect(JSON.stringify(res.body)).not.toContain(invalid); + }); + it("refuses an unknown action and an unknown bot", async () => { const bad = await api("POST", `/api/bots/${botId}/computer/control`, { action: "hijack" }); expect(bad.status).toBe(400); diff --git a/server/index.ts b/server/index.ts index 15f278122..6e80578e1 100644 --- a/server/index.ts +++ b/server/index.ts @@ -27,6 +27,7 @@ import * as box from "./box.ts"; import { cloudBackendChangeError, vpsAliasChangeError } from "./cloud-backend.ts"; import * as composio from "./composio.ts"; import { chiefOfStaffSystemPrompt } from "./chief-of-staff.ts"; +import { openMausStatusSystemPrompt } from "./openmaus-status-capsule.ts"; import { containerComputerAction, containerComputerExists, @@ -195,6 +196,7 @@ function connectedAppsIntegration(botId: string, threadId: string) { const computerControl = new ComputerControl((botId, snapshot) => { broadcast({ kind: "computer-control", botId, held: snapshot.held, helpReason: snapshot.helpReason }); }); +const controlLeaseIdSchema = z.string().min(16).max(120).regex(/^[A-Za-z0-9_-]+$/); /** The loopback endpoint a bot's computer proxy polls before acting. */ function controlIntegration(botId: string) { @@ -1604,7 +1606,12 @@ async function startTurn( ) : []; const coordinationPrompt = bot.chiefOfStaff - ? chiefOfStaffSystemPrompt(bot.id, store.bots, Boolean(integrations.agents)) + ? chiefOfStaffSystemPrompt( + bot.id, + store.bots, + Boolean(integrations.agents), + openMausStatusSystemPrompt(), + ) : integrations.agents ? "You can work with the user's other bots through the agents tools — list_bots shows who's available, ask_bot sends one of them a message and returns their reply." : ""; @@ -4248,6 +4255,26 @@ const server = createServer(async (req, res) => { } const body = await readBody(req); const action = String(body.action ?? ""); + const leaseResult = + body.controlLeaseId === undefined + ? null + : controlLeaseIdSchema.safeParse(body.controlLeaseId); + if (leaseResult && !leaseResult.success) { + return json(res, 400, { error: "controlLeaseId is invalid" }); + } + const controlLeaseId = leaseResult?.data; + if (action === "take" && controlLeaseId) { + const result = computerControl.acquireLease(bot.id, controlLeaseId); + return json(res, 200, { + ...result.snapshot, + owned: result.owned, + acquired: result.acquired, + }); + } + if (action === "release" && controlLeaseId) { + const result = computerControl.releaseLease(bot.id, controlLeaseId); + return json(res, 200, { ...result.snapshot, released: result.released }); + } if (action === "take") return json(res, 200, computerControl.take(bot.id)); if (action === "release") return json(res, 200, computerControl.release(bot.id)); if (action === "dismiss-help") return json(res, 200, computerControl.dismissHelp(bot.id)); diff --git a/server/openmaus-status-capsule.test.ts b/server/openmaus-status-capsule.test.ts new file mode 100644 index 000000000..a0138896b --- /dev/null +++ b/server/openmaus-status-capsule.test.ts @@ -0,0 +1,301 @@ +import { createHash } from "node:crypto"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { openMausStatusSystemPrompt, readOpenMausStatus } from "./openmaus-status-capsule.ts"; +import type { JsonObject, JsonValue } from "./schema.ts"; + +const NOW = new Date("2026-08-22T06:30:00Z"); +const OLD_SINGLE_VIEW_SHA = `sha256:${"1".repeat(64)}`; +const DUAL_VIEW_SHA = `sha256:${"2".repeat(64)}`; +const roots: string[] = []; +const jsonObjectSchema = z.record(z.string(), z.custom()); + +interface TestCapsule extends JsonObject { + schema: string; + observed_at: string; + fresh_until: string; + ttl_seconds: number; + source_sha256: string | null; + dual_view_sha256: string | null; + refresh_status: string; + runtime_state: string; + mode: string; + max_instances: number | null; + ready_count: number; + slots: JsonObject[]; + ui: JsonObject; +} + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function canonicalValue(value: JsonValue): JsonValue { + if (Array.isArray(value)) return value.map(canonicalValue); + const parsedObject = jsonObjectSchema.safeParse(value); + if (!parsedObject.success) return value; + const sorted: JsonObject = {}; + for (const key of Object.keys(parsedObject.data).sort()) { + const child = parsedObject.data[key]; + if (child !== undefined) sorted[key] = canonicalValue(child); + } + return sorted; +} + +function canonical(value: JsonValue): string { + return JSON.stringify(canonicalValue(value)); +} + +function sign(value: TestCapsule): TestCapsule { + const signed: TestCapsule = { ...value }; + delete signed.receipt_sha256; + signed.receipt_sha256 = `sha256:${createHash("sha256").update(canonical(signed)).digest("hex")}`; + return signed; +} + +function ui(twoUp: boolean): JsonObject { + return { + two_up: twoUp, + max_visible: twoUp ? 2 : 1, + max_interactive: 1, + default_watch_only: twoUp, + }; +} + +function successCapsule(options: { + source?: string | null; + expected?: string | null; + twoUp?: boolean; +} = {}): TestCapsule { + const source = options.source === undefined ? DUAL_VIEW_SHA : options.source; + const expected = options.expected === undefined ? DUAL_VIEW_SHA : options.expected; + const twoUp = options.twoUp ?? (source !== null && source === expected); + return sign({ + schema: "aos.openmausbot_status.v1", + observed_at: "2026-08-22T06:30:00Z", + fresh_until: "2026-08-22T06:35:00Z", + ttl_seconds: 300, + source_sha256: source, + dual_view_sha256: expected, + refresh_status: "success", + runtime_state: "degraded", + mode: "per-bot", + max_instances: 2, + ready_count: 0, + slots: [ + { + slot: "vm-1", + container: "missing", + readiness: "not_ready", + network: "unknown", + security: "unknown", + persistence: "unknown", + }, + { + slot: "vm-2", + container: "missing", + readiness: "not_ready", + network: "unknown", + security: "unknown", + persistence: "unknown", + }, + ], + ui: ui(twoUp), + }); +} + +function failedCapsule( + reason = "config_unavailable", + source: string | null = DUAL_VIEW_SHA, + expected: string | null = DUAL_VIEW_SHA, +): TestCapsule { + return sign({ + schema: "aos.openmausbot_status.v1", + observed_at: "2026-08-22T06:30:00Z", + fresh_until: "2026-08-22T06:35:00Z", + ttl_seconds: 300, + source_sha256: source, + dual_view_sha256: expected, + refresh_status: "failed", + failure_reason: reason, + runtime_state: "unknown", + mode: "unknown", + max_instances: null, + ready_count: 0, + slots: [], + ui: ui(false), + }); +} + +function cachePath(capsule: TestCapsule): string { + const root = mkdtempSync(join(tmpdir(), "openmaus-status-")); + roots.push(root); + const parent = join(root, "openmausbot"); + mkdirSync(parent, { mode: 0o700 }); + chmodSync(parent, 0o700); + const path = join(parent, "latest.json"); + writeFileSync(path, `${canonical(capsule)}\n`, { mode: 0o600 }); + chmodSync(path, 0o600); + return path; +} + +describe("readOpenMausStatus", () => { + it("projects only fresh normalized two-VM capability data", () => { + const capsule = successCapsule(); + // Cross-language receipt produced by scripts/aos_openmausbot_status.py + // for this exact normalized fixture. + expect(capsule.receipt_sha256).toBe( + "sha256:2f76115fcbf37dfc5406d4a7a460c5e3016ff87184cd9e314bf4cc11022e2d7c", + ); + const path = cachePath(capsule); + + const status = readOpenMausStatus({ cachePath: path, now: new Date(NOW.getTime() + 1_000) }); + + expect(status).toMatchObject({ + freshness: "fresh", + runtimeState: "degraded", + mode: "per-bot", + maxInstances: 2, + readyCount: 0, + sourceSha256: DUAL_VIEW_SHA, + dualViewSha256: DUAL_VIEW_SHA, + ui: { + twoUp: true, + maxVisible: 2, + maxInteractive: 1, + defaultWatchOnly: true, + oneActiveController: true, + }, + }); + expect(status.slots).toHaveLength(2); + expect(Object.keys(status.slots[0]).sort()).toEqual( + ["container", "network", "persistence", "readiness", "security", "slot"], + ); + const prompt = openMausStatusSystemPrompt({ cachePath: path, now: new Date(NOW.getTime() + 1_000) }); + expect(prompt).toContain("freshness=fresh"); + expect(prompt).toContain("ui.two_up=true"); + expect(prompt).toContain(`source_sha256=${DUAL_VIEW_SHA}`); + expect(prompt).toContain(`accepted_dual_view_sha256=${DUAL_VIEW_SHA}`); + expect(prompt).toContain("source_match=true"); + expect(prompt).toContain("one_active_controller=true"); + expect(prompt).not.toMatch(/viewer_url|password|bot-alpha|Private VM Alpha|workspace_path|held=/); + }); + + it.each([ + ["old single-view per-bot app", OLD_SINGLE_VIEW_SHA, DUAL_VIEW_SHA, "source_hash_mismatch"], + ["mismatched dual build", `sha256:${"3".repeat(64)}`, DUAL_VIEW_SHA, "source_hash_mismatch"], + ["unavailable installed hash", null, DUAL_VIEW_SHA, "source_hash_unavailable"], + ["unavailable expected hash", DUAL_VIEW_SHA, null, "source_hash_unavailable"], + ])("turns %s state into unknown", (_label, source, expected, reason) => { + const path = cachePath(failedCapsule(reason, source, expected)); + + const status = readOpenMausStatus({ cachePath: path, now: new Date(NOW.getTime() + 1_000) }); + + expect(status.freshness).toBe("fresh"); + expect(status.reason).toBe("refresh_failed"); + expect(status.runtimeState).toBe("unknown"); + expect(status.readyCount).toBe(0); + expect(status.slots).toEqual([]); + expect(status.ui).toMatchObject({ twoUp: false, maxVisible: 1, defaultWatchOnly: false }); + }); + + it("rejects a signed success capsule whose installed and accepted hashes differ", () => { + const path = cachePath(successCapsule({ + source: OLD_SINGLE_VIEW_SHA, + expected: DUAL_VIEW_SHA, + twoUp: false, + })); + + expect(readOpenMausStatus({ cachePath: path, now: NOW })).toMatchObject({ + freshness: "unknown", + reason: "invalid", + runtimeState: "unknown", + readyCount: 0, + slots: [], + ui: { twoUp: false }, + }); + }); + + it("accepts more configured VM bots than the simultaneous instance limit", () => { + const capsule = successCapsule({ twoUp: false }); + capsule.max_instances = 1; + const path = cachePath(sign(capsule)); + + const status = readOpenMausStatus({ cachePath: path, now: new Date(NOW.getTime() + 1_000) }); + + expect(status).toMatchObject({ + freshness: "fresh", + maxInstances: 1, + readyCount: 0, + ui: { twoUp: false, maxVisible: 1 }, + }); + expect(status.slots).toHaveLength(2); + }); + + it("turns stale, future-skewed, failed, and receipt-tampered state into unknown", () => { + const path = cachePath(successCapsule()); + expect(readOpenMausStatus({ cachePath: path, now: new Date("2026-08-22T06:35:00Z") })).toMatchObject({ + freshness: "stale", reason: "stale", runtimeState: "unknown", readyCount: 0, + ui: { twoUp: false }, + }); + expect(readOpenMausStatus({ cachePath: path, now: new Date("2026-08-22T06:29:59Z") })).toMatchObject({ + freshness: "unknown", reason: "clock_skew", runtimeState: "unknown", readyCount: 0, + }); + + const failurePath = cachePath(failedCapsule()); + expect(readOpenMausStatus({ cachePath: failurePath, now: new Date(NOW.getTime() + 1_000) })).toMatchObject({ + freshness: "fresh", reason: "refresh_failed", runtimeState: "unknown", readyCount: 0, + ui: { twoUp: false }, + }); + + const tampered = successCapsule(); + tampered.ready_count = 1; + const tamperedPath = cachePath(tampered); + expect(readOpenMausStatus({ cachePath: tamperedPath, now: new Date(NOW.getTime() + 1_000) })).toMatchObject({ + freshness: "unknown", reason: "invalid", runtimeState: "unknown", readyCount: 0, + }); + }); + + it("rejects signed extra fields, insecure modes, and same-path symlinks", () => { + const extra = successCapsule(); + extra.viewer_url = "http://127.0.0.1:62001/private"; + const extraPath = cachePath(sign(extra)); + expect(readOpenMausStatus({ cachePath: extraPath, now: NOW }).reason).toBe("invalid"); + + const insecurePath = cachePath(successCapsule()); + chmodSync(insecurePath, 0o644); + expect(readOpenMausStatus({ cachePath: insecurePath, now: NOW }).reason).toBe("missing_or_insecure"); + + const targetPath = cachePath(successCapsule()); + const linkPath = join(dirname(targetPath), "latest-link.json"); + symlinkSync(targetPath, linkPath); + expect(readOpenMausStatus({ cachePath: linkPath, now: NOW }).reason).toBe("missing_or_insecure"); + }); + + it("rejects a non-0700 parent and a symlinked parent", () => { + const looseParentPath = cachePath(successCapsule()); + chmodSync(dirname(looseParentPath), 0o755); + expect(readOpenMausStatus({ cachePath: looseParentPath, now: NOW }).reason).toBe("missing_or_insecure"); + + const targetPath = cachePath(successCapsule()); + const linkRoot = mkdtempSync(join(tmpdir(), "openmaus-status-parent-link-")); + roots.push(linkRoot); + const linkedParent = join(linkRoot, "openmausbot"); + symlinkSync(dirname(targetPath), linkedParent, "dir"); + expect( + readOpenMausStatus({ cachePath: join(linkedParent, "latest.json"), now: NOW }).reason, + ).toBe("missing_or_insecure"); + }); +}); diff --git a/server/openmaus-status-capsule.ts b/server/openmaus-status-capsule.ts new file mode 100644 index 000000000..fd332040a --- /dev/null +++ b/server/openmaus-status-capsule.ts @@ -0,0 +1,379 @@ +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; + +import { z } from "zod"; + +import { parseJson, type JsonObject, type JsonValue } from "./schema.ts"; + +const SCHEMA = "aos.openmausbot_status.v1"; +const TTL_SECONDS = 300; +const MAX_CACHE_BYTES = 16_384; +const DIGEST = /^sha256:[a-f0-9]{64}$/; +const UTC_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/; + +const digestSchema = z.string().regex(DIGEST); +const slotSchema = z.object({ + slot: z.string().regex(/^vm-[1-4]$/), + container: z.enum(["running", "stopped", "missing"]), + readiness: z.enum(["ready", "not_ready"]), + network: z.enum(["loopback", "unsafe", "unknown"]), + security: z.enum(["hardened", "unsafe", "unknown"]), + persistence: z.enum(["durable", "unsafe", "unknown"]), +}).strict(); +const uiSchema = z.object({ + two_up: z.boolean(), + max_visible: z.union([z.literal(1), z.literal(2)]), + max_interactive: z.literal(1), + default_watch_only: z.boolean(), +}).strict(); +const capsuleSchema = z.object({ + schema: z.literal(SCHEMA), + observed_at: z.string().regex(UTC_TIMESTAMP), + fresh_until: z.string().regex(UTC_TIMESTAMP), + ttl_seconds: z.literal(TTL_SECONDS), + source_sha256: digestSchema.nullable(), + dual_view_sha256: digestSchema.nullable(), + receipt_sha256: digestSchema, + refresh_status: z.enum(["success", "failed"]), + failure_reason: z.enum([ + "config_unavailable", + "bots_unavailable", + "vm_status_unavailable", + "capacity_exceeded", + "source_hash_unavailable", + "source_hash_mismatch", + ]).optional(), + runtime_state: z.enum(["ready", "degraded", "unknown"]), + mode: z.enum(["shared", "per-bot", "unknown"]), + max_instances: z.number().int().min(1).max(4).nullable(), + ready_count: z.number().int().min(0).max(4), + slots: z.array(slotSchema).max(4), + ui: uiSchema, +}).strict(); + +const jsonObjectSchema = z.record(z.string(), z.custom()); + +export const OPENMAUS_STATUS_CACHE_PATH = join( + homedir(), + ".local/state/aos-session-bridge/openmausbot/latest.json", +); + +type OpenMausSlot = z.output; +type OpenMausUi = z.output; +type OpenMausCapsule = z.output; + +export interface OpenMausStatusDigest { + schema: typeof SCHEMA; + freshness: "fresh" | "stale" | "unknown"; + reason?: "missing_or_insecure" | "invalid" | "clock_skew" | "stale" | "refresh_failed"; + observedAt?: string; + receiptSha256?: string; + sourceSha256?: string; + dualViewSha256?: string; + runtimeState: "ready" | "degraded" | "unknown"; + mode: "shared" | "per-bot" | "unknown"; + maxInstances: number | null; + readyCount: number; + slots: OpenMausSlot[]; + ui: { + twoUp: boolean; + maxVisible: 1 | 2; + maxInteractive: 1; + defaultWatchOnly: boolean; + oneActiveController: true; + }; +} + +export interface OpenMausStatusReadOptions { + cachePath?: string; + now?: Date; +} + +function canonicalValue(value: JsonValue): JsonValue { + if (Array.isArray(value)) return value.map(canonicalValue); + const parsedObject = jsonObjectSchema.safeParse(value); + if (!parsedObject.success) return value; + const sorted: JsonObject = {}; + for (const key of Object.keys(parsedObject.data).sort()) { + const child = parsedObject.data[key]; + if (child !== undefined) sorted[key] = canonicalValue(child); + } + return sorted; +} + +function canonical(value: JsonValue): string { + return JSON.stringify(canonicalValue(value)); +} + +function sha256(value: string): string { + return `sha256:${createHash("sha256").update(value, "utf8").digest("hex")}`; +} + +function timestamp(value: string): number | null { + const parsed = Date.parse(value); + if (!Number.isFinite(parsed)) return null; + if (new Date(parsed).toISOString().replace(".000Z", "Z") !== value) return null; + return parsed; +} + +function expectedUi(twoUp: boolean): OpenMausUi { + return { + two_up: twoUp, + max_visible: twoUp ? 2 : 1, + max_interactive: 1, + default_watch_only: twoUp, + }; +} + +function sameUi(left: OpenMausUi, right: OpenMausUi): boolean { + return ( + left.two_up === right.two_up && + left.max_visible === right.max_visible && + left.max_interactive === right.max_interactive && + left.default_watch_only === right.default_watch_only + ); +} + +function unsignedDocument(capsule: OpenMausCapsule): JsonObject { + const document: JsonObject = { + schema: capsule.schema, + observed_at: capsule.observed_at, + fresh_until: capsule.fresh_until, + ttl_seconds: capsule.ttl_seconds, + source_sha256: capsule.source_sha256, + dual_view_sha256: capsule.dual_view_sha256, + refresh_status: capsule.refresh_status, + runtime_state: capsule.runtime_state, + mode: capsule.mode, + max_instances: capsule.max_instances, + ready_count: capsule.ready_count, + slots: capsule.slots.map((slot): JsonObject => ({ + slot: slot.slot, + container: slot.container, + readiness: slot.readiness, + network: slot.network, + security: slot.security, + persistence: slot.persistence, + })), + ui: { + two_up: capsule.ui.two_up, + max_visible: capsule.ui.max_visible, + max_interactive: capsule.ui.max_interactive, + default_watch_only: capsule.ui.default_watch_only, + }, + }; + if (capsule.failure_reason !== undefined) document.failure_reason = capsule.failure_reason; + return document; +} + +function validSlotState(slot: OpenMausSlot): boolean { + return ( + slot.readiness !== "ready" || + (slot.container === "running" && + slot.network === "loopback" && + slot.security === "hardened" && + slot.persistence === "durable") + ); +} + +function validateCapsule(value: JsonValue): OpenMausCapsule | null { + const parsed = capsuleSchema.safeParse(value); + if (!parsed.success) return null; + const capsule = parsed.data; + const observed = timestamp(capsule.observed_at); + const freshUntil = timestamp(capsule.fresh_until); + if (observed === null || freshUntil === null || freshUntil - observed !== TTL_SECONDS * 1000) return null; + if (capsule.receipt_sha256 !== sha256(canonical(unsignedDocument(capsule)))) return null; + if (!capsule.slots.every((slot, index) => slot.slot === `vm-${index + 1}` && validSlotState(slot))) { + return null; + } + const readyCount = capsule.slots.filter((slot) => slot.readiness === "ready").length; + if (capsule.ready_count !== readyCount) return null; + const twoUp = Boolean( + capsule.refresh_status === "success" && + capsule.mode === "per-bot" && + capsule.max_instances !== null && + capsule.max_instances >= 2 && + capsule.source_sha256 !== null && + capsule.dual_view_sha256 !== null && + capsule.source_sha256 === capsule.dual_view_sha256, + ); + if (!sameUi(capsule.ui, expectedUi(twoUp))) return null; + + if (capsule.refresh_status === "failed") { + if ( + capsule.failure_reason === undefined || + capsule.runtime_state !== "unknown" || + capsule.mode !== "unknown" || + capsule.max_instances !== null || + capsule.ready_count !== 0 || + capsule.slots.length !== 0 || + !sameUi(capsule.ui, expectedUi(false)) + ) return null; + } else { + const expectedRuntime = capsule.slots.length > 0 && readyCount === capsule.slots.length ? "ready" : "degraded"; + if ( + capsule.failure_reason !== undefined || + capsule.runtime_state !== expectedRuntime || + (capsule.mode !== "shared" && capsule.mode !== "per-bot") || + capsule.max_instances === null || + readyCount > capsule.max_instances || + capsule.source_sha256 === null || + capsule.dual_view_sha256 === null || + capsule.source_sha256 !== capsule.dual_view_sha256 + ) return null; + } + return capsule; +} + +function privateCache(path: string): Buffer | null { + const uid = process.getuid?.(); + if (uid === undefined) return null; + let descriptor: number | null = null; + try { + const parentStatus = lstatSync(dirname(path)); + const fileStatus = lstatSync(path); + if ( + parentStatus.isSymbolicLink() || + !parentStatus.isDirectory() || + parentStatus.uid !== uid || + (parentStatus.mode & 0o777) !== 0o700 + ) return null; + if ( + fileStatus.isSymbolicLink() || + !fileStatus.isFile() || + fileStatus.uid !== uid || + (fileStatus.mode & 0o777) !== 0o600 || + fileStatus.size > MAX_CACHE_BYTES + ) return null; + descriptor = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + const openedStatus = fstatSync(descriptor); + if ( + !openedStatus.isFile() || + openedStatus.uid !== uid || + (openedStatus.mode & 0o777) !== 0o600 || + openedStatus.dev !== fileStatus.dev || + openedStatus.ino !== fileStatus.ino || + openedStatus.size > MAX_CACHE_BYTES + ) return null; + const contents = readFileSync(descriptor); + return contents.length <= MAX_CACHE_BYTES ? contents : null; + } catch { + return null; + } finally { + if (descriptor !== null) closeSync(descriptor); + } +} + +function unknownDigest( + reason: NonNullable, + freshness: OpenMausStatusDigest["freshness"] = "unknown", + receipt?: Pick, +): OpenMausStatusDigest { + return { + schema: SCHEMA, + freshness, + reason, + ...receipt, + runtimeState: "unknown", + mode: "unknown", + maxInstances: null, + readyCount: 0, + slots: [], + ui: { + twoUp: false, + maxVisible: 1, + maxInteractive: 1, + defaultWatchOnly: false, + oneActiveController: true, + }, + }; +} + +export function readOpenMausStatus( + options: OpenMausStatusReadOptions = {}, +): OpenMausStatusDigest { + const raw = privateCache(options.cachePath ?? OPENMAUS_STATUS_CACHE_PATH); + if (raw === null) return unknownDigest("missing_or_insecure"); + let capsule: OpenMausCapsule | null; + try { + capsule = validateCapsule(parseJson(raw.toString("utf8"))); + } catch { + capsule = null; + } + if (capsule === null) return unknownDigest("invalid"); + const observed = timestamp(capsule.observed_at)!; + const freshUntil = timestamp(capsule.fresh_until)!; + const now = (options.now ?? new Date()).getTime(); + const receipt = { observedAt: capsule.observed_at, receiptSha256: capsule.receipt_sha256 }; + if (!Number.isFinite(now) || now < observed) return unknownDigest("clock_skew"); + if (now >= freshUntil) return unknownDigest("stale", "stale", receipt); + if (capsule.refresh_status === "failed") return unknownDigest("refresh_failed", "fresh", receipt); + const result: OpenMausStatusDigest = { + schema: SCHEMA, + freshness: "fresh", + ...receipt, + runtimeState: capsule.runtime_state, + mode: capsule.mode, + maxInstances: capsule.max_instances, + readyCount: capsule.ready_count, + slots: capsule.slots, + ui: { + twoUp: capsule.ui.two_up, + maxVisible: capsule.ui.max_visible, + maxInteractive: 1, + defaultWatchOnly: capsule.ui.default_watch_only, + oneActiveController: true, + }, + }; + if (capsule.source_sha256 !== null) result.sourceSha256 = capsule.source_sha256; + if (capsule.dual_view_sha256 !== null) result.dualViewSha256 = capsule.dual_view_sha256; + return result; +} + +export function openMausStatusSystemPrompt(options: OpenMausStatusReadOptions = {}): string { + const status = readOpenMausStatus(options); + const receipt = [ + status.observedAt ? `observed_at=${status.observedAt}` : null, + status.receiptSha256 ? `receipt_sha256=${status.receiptSha256}` : null, + status.sourceSha256 ? `source_sha256=${status.sourceSha256}` : null, + status.dualViewSha256 ? `accepted_dual_view_sha256=${status.dualViewSha256}` : null, + status.sourceSha256 && status.dualViewSha256 + ? `source_match=${status.sourceSha256 === status.dualViewSha256}` + : null, + ].filter(Boolean).join("; "); + const runtime = status.freshness === "fresh" && status.reason === undefined + ? [ + `runtime_state=${status.runtimeState}`, + `mode=${status.mode}`, + `maximum_instances=${status.maxInstances}`, + `ready_count=${status.readyCount}`, + ].join("; ") + : "runtime_state=unknown; mode=unknown; maximum_instances=unknown; ready_count=0"; + const slots = status.slots.length + ? status.slots + .map((slot) => + `${slot.slot}(container=${slot.container},readiness=${slot.readiness},network=${slot.network},security=${slot.security},persistence=${slot.persistence})` + ) + .join(",") + : "none"; + return [ + "TRUSTED OPENMAUSBOT STATUS (read-only, validated, no transcript or credential data):", + `schema=${status.schema}; freshness=${status.freshness}${status.reason ? `; reason=${status.reason}` : ""}`, + receipt || "receipt=unavailable", + runtime, + `anonymous_slots=${slots}`, + `ui.two_up=${status.ui.twoUp}; ui.max_visible=${status.ui.maxVisible}; ui.max_interactive=1; ui.default_watch_only=${status.ui.defaultWatchOnly}; one_active_controller=true`, + "Opening a viewer or two-up workspace never starts or provisions a VM. Only one pane may be interactive at a time; switching control must release the previous pane before activating the next.", + "Treat missing, stale, failed, clock-skewed, malformed, or receipt-hash-mismatched runtime data as unknown. Do not infer bot identities, viewer URLs, paths, messages, models, accounts, or credentials from this block.", + ].join("\n"); +} diff --git a/src/App.tsx b/src/App.tsx index 7b959a8aa..2bbb991ae 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -16,6 +16,7 @@ import { DesktopCapabilitiesProvider } from "@/components/DesktopCapabilities"; import { RoutinesPage } from "@/components/RoutinesPage"; import { NoEngines } from "@/components/NoEngines"; import { CommandPalette } from "@/components/CommandPalette"; +import { LocalVmWorkspace } from "@/components/LocalVmWorkspace"; function Shell() { const { state, dispatch } = useStore(); @@ -25,6 +26,8 @@ function Shell() { // turn the aside into a containing block for its fixed descendants (see // Sidebar.tsx's className comment). const [drawerOpen, setDrawerOpen] = useState(false); + const [paletteOpen, setPaletteOpen] = useState(false); + const [localVmWorkspaceBotId, setLocalVmWorkspaceBotId] = useState(null); const menuButtonRef = useRef(null); const group = state.groups.find((g) => g.id === state.selectedId); const bot = group ? undefined : (state.bots.find((b) => b.id === state.selectedId) ?? state.bots[0]); @@ -77,6 +80,35 @@ function Shell() { setDrawerOpen(false); }, [state.selectedId, state.activeView, state.pluginsOpen, state.settingsOpen]); + useEffect(() => { + if ( + localVmWorkspaceBotId && + (state.activeView !== "chat" || state.selectedId !== localVmWorkspaceBotId) + ) { + setLocalVmWorkspaceBotId(null); + } + }, [localVmWorkspaceBotId, state.activeView, state.selectedId]); + + const openLocalVmWorkspace = (botId: string) => { + dispatch({ type: "toggleComputer", open: false }); + setLocalVmWorkspaceBotId(botId); + }; + + const openComputerFromWorkspace = (botId: string) => { + setLocalVmWorkspaceBotId(null); + dispatch({ type: "select", id: botId }); + dispatch({ type: "toggleComputer", open: true }); + }; + + const nativeViewOverlayOpen = + drawerOpen || + paletteOpen || + state.settingsOpen || + state.computerOpen || + state.inspectorOpen || + state.appSettingsOpen || + state.pluginsOpen; + return (
{/* fixed-position popup, bottom-left — outside the layout flow */} @@ -108,6 +140,13 @@ function Shell() { /> {state.activeView === "routines" ? ( + ) : localVmWorkspaceBotId ? ( + setLocalVmWorkspaceBotId(null)} + onOpenComputer={openComputerFromWorkspace} + /> ) : noEngines ? ( ) : group ? ( @@ -128,13 +167,15 @@ function Shell() { )} {state.settingsOpen && bot && } - {state.computerOpen && bot && } + {state.computerOpen && bot && ( + + )} {state.inspectorOpen && bot && } {state.appSettingsOpen && } {state.pluginsOpen && } {/* mounted after the modals: same z-50 tier, so DOM order keeps the palette on top when one of them is open underneath */} - +
); diff --git a/src/components/CommandPalette.tsx b/src/components/CommandPalette.tsx index ccdd7d733..2d0c8b257 100644 --- a/src/components/CommandPalette.tsx +++ b/src/components/CommandPalette.tsx @@ -14,7 +14,7 @@ type PaletteEntry = | { kind: "room"; group: Group } | { kind: "message"; hit: SearchHit }; -export function CommandPalette() { +export function CommandPalette({ onOpenChange }: { onOpenChange?: (open: boolean) => void }) { const { state, dispatch } = useStore(); const [open, setOpen] = useState(false); const [query, setQuery] = useState(""); @@ -44,6 +44,10 @@ export function CommandPalette() { setCursor(0); }, [open]); + useEffect(() => { + onOpenChange?.(open); + }, [onOpenChange, open]); + const q = query.trim().toLowerCase(); // Same debounce pattern as the sidebar search: names answer instantly diff --git a/src/components/ComputerPanel.tsx b/src/components/ComputerPanel.tsx index ab42106c7..2d9ca7a3c 100644 --- a/src/components/ComputerPanel.tsx +++ b/src/components/ComputerPanel.tsx @@ -8,6 +8,7 @@ import { useEffect, useRef, useState } from "react"; import { CalendarDays, CalendarClock, + Columns2, Hand, Loader2, Monitor, @@ -104,7 +105,13 @@ function nextRunLabel(at: number | null) { return `${sameDay ? "Today" : date.toLocaleDateString([], { month: "short", day: "numeric" })}, ${date.toLocaleTimeString([], { hour: "numeric", minute: "2-digit" })}`; } -export function ComputerPanel({ bot }: { bot: Bot }) { +export function ComputerPanel({ + bot, + onOpenVmWorkspace, +}: { + bot: Bot; + onOpenVmWorkspace?: (botId: string) => void; +}) { const { state, dispatch } = useStore(); const { capabilities, ready: capabilitiesReady } = useDesktopCapabilities(); const localAvailable = capabilities.localComputer.available; @@ -808,6 +815,21 @@ export function ComputerPanel({ bot }: { bot: Bot }) { )} + {vmStatus?.mode === "per-bot" && + window.ogb?.desktopWorkspace && + onOpenVmWorkspace && ( + + )} + {/* Who is driving — take the wheel / hand it back */} {(phase === "ready" || phase === "vm") && control.helpReason && !control.held && (
diff --git a/src/components/LocalVmWorkspace.tsx b/src/components/LocalVmWorkspace.tsx new file mode 100644 index 000000000..3c744832a --- /dev/null +++ b/src/components/LocalVmWorkspace.tsx @@ -0,0 +1,756 @@ +import { + AlertTriangle, + Hand, + Loader2, + Monitor, + RefreshCw, + X, +} from "lucide-react"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + type RefObject, +} from "react"; +import { api, useStore, type Action, type Bot } from "@/state/store"; +import { cn } from "@/lib/cn"; +import { + initialLocalVmWorkspaceSlots, + nativeViewOverlayIntersects, + readyLocalVmViewerUrl, + reconcileLocalVmWorkspaceSlots, + releaseLocalVmWorkspaceControl, + sanitizeLocalVmWorkspaceStatus, + selectLocalVmWorkspaceSlot, + switchLocalVmWorkspaceControl, + type LocalVmWorkspaceControlPort, + type LocalVmWorkspaceControlSnapshot, + type LocalVmWorkspaceSlots, + type LocalVmWorkspaceStatus, +} from "@/lib/local-vm-workspace"; +import { z } from "zod"; + +const SLOT_CONTEXTS = ["local-vm-workspace:left", "local-vm-workspace:right"] as const; + +type WorkspaceDispatch = (action: Action) => void; + +const controlSnapshotSchema = z.object({ + held: z.boolean(), + helpReason: z.string().nullable(), + owned: z.boolean().optional(), + acquired: z.boolean().optional(), + released: z.boolean().optional(), +}); + +interface LocalVmWorkspaceProps { + primaryBotId: string; + overlayOpen: boolean; + onClose(): void; + onOpenComputer(botId: string): void; +} + +async function requestComputerControl( + botId: string, + action: "take" | "release", + controlLeaseId: string, +): Promise { + const result = await api(`/api/bots/${botId}/computer/control`, { + method: "POST", + body: JSON.stringify({ action, controlLeaseId }), + }); + const parsed = controlSnapshotSchema.safeParse(result); + if (!parsed.success) throw new Error("invalid-control-snapshot"); + return parsed.data; +} + +async function readComputerControl(botId: string): Promise { + const parsed = controlSnapshotSchema.safeParse( + await api(`/api/bots/${botId}/computer/control`), + ); + if (!parsed.success) throw new Error("invalid-control-snapshot"); + return parsed.data; +} + +function bestEffortRelease(botId: string, controlLeaseId: string) { + void fetch(`/api/bots/${botId}/computer/control`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ action: "release", controlLeaseId }), + keepalive: true, + }).catch(() => {}); +} + +function dispatchControl( + dispatch: WorkspaceDispatch, + botId: string, + snapshot: LocalVmWorkspaceControlSnapshot, +) { + dispatch({ + type: "computerControl", + botId, + held: snapshot.held, + helpReason: snapshot.helpReason, + }); +} + +function elementBounds(ref: RefObject): DesktopWorkspaceBounds | null { + const rect = ref.current?.getBoundingClientRect(); + if (!rect || rect.width < 1 || rect.height < 1) return null; + return { + x: Math.round(rect.left), + y: Math.round(rect.top), + width: Math.round(rect.width), + height: Math.round(rect.height), + }; +} + +const NATIVE_VIEW_OVERLAY_SELECTOR = [ + '[aria-modal="true"]', + '[role="dialog"]', + '[role="menu"]', + "[popover]", + "[data-native-view-overlay]", + ".fixed", + ".absolute", +].join(","); + +/** Native views always paint above the renderer. Detect every visible + * positioned overlay or popover that intersects a pane, including portal + * content such as sidebar menus and the update banner. */ +function rendererOverlayIntersectsNativeView() { + const hosts = [...document.querySelectorAll("[data-native-view-host]")]; + const hostRects = hosts + .map((host) => host.getBoundingClientRect()) + .filter((rect) => rect.width > 0 && rect.height > 0); + if (hostRects.length === 0) return false; + + const candidates = [...document.querySelectorAll(NATIVE_VIEW_OVERLAY_SELECTOR)] + .filter( + (candidate) => + !hosts.some( + (host) => candidate === host || candidate.contains(host) || host.contains(candidate), + ), + ) + .map((candidate) => { + const style = window.getComputedStyle(candidate); + const explicitlyOverlay = + candidate.matches( + '[aria-modal="true"], [role="dialog"], [role="menu"], [popover], [data-native-view-overlay]', + ); + const zIndex = Number.parseInt(style.zIndex, 10); + return { + rect: candidate.getBoundingClientRect(), + explicit: explicitlyOverlay, + visible: + style.display !== "none" && + style.visibility !== "hidden" && + Number(style.opacity) !== 0, + zIndex: Number.isFinite(zIndex) ? zIndex : null, + }; + }); + return nativeViewOverlayIntersects(hostRects, candidates); +} + +function useNativeViewObscured(explicit: boolean) { + const [domOverlay, setDomOverlay] = useState(false); + useEffect(() => { + let frame = 0; + const read = () => setDomOverlay(rendererOverlayIntersectsNativeView()); + const scheduleRead = () => { + if (frame) return; + frame = requestAnimationFrame(() => { + frame = 0; + read(); + }); + }; + read(); + const observer = new MutationObserver(scheduleRead); + observer.observe(document.body, { + subtree: true, + childList: true, + attributes: true, + attributeFilter: ["class", "style", "aria-hidden", "aria-modal", "role", "open", "popover"], + }); + const resizeObserver = new ResizeObserver(scheduleRead); + resizeObserver.observe(document.body); + for (const host of document.querySelectorAll("[data-native-view-host]")) { + resizeObserver.observe(host); + } + const overlayEvents = [ + "toggle", + "animationstart", + "animationend", + "transitionstart", + "transitionend", + ] as const; + for (const eventName of overlayEvents) { + document.addEventListener(eventName, scheduleRead, true); + } + window.addEventListener("resize", scheduleRead); + window.addEventListener("scroll", scheduleRead, true); + return () => { + if (frame) cancelAnimationFrame(frame); + observer.disconnect(); + resizeObserver.disconnect(); + for (const eventName of overlayEvents) { + document.removeEventListener(eventName, scheduleRead, true); + } + window.removeEventListener("resize", scheduleRead); + window.removeEventListener("scroll", scheduleRead, true); + }; + }, []); + return explicit || domOverlay; +} + +function statusLabel(status: LocalVmWorkspaceStatus | null, nativeStatus: DesktopWorkspaceState["status"]) { + if (!status) return "Checking VM"; + if (status.container === "missing") return "VM not created"; + if (status.container === "stopped") return "VM stopped"; + if (!status.ready) return "VM unavailable"; + if (nativeStatus === "error") return "Viewer unavailable"; + if (nativeStatus !== "ready") return "Connecting viewer"; + return "Live · watch-only"; +} + +interface LocalVmPaneProps { + index: 0 | 1; + bot: Bot | null; + bots: Bot[]; + otherBotId: string | null; + obscured: boolean; + active: boolean; + heldElsewhere: boolean; + controlPending: boolean; + onSelect(botId: string | null): void; + onTake(botId: string): void; + onRelease(): void; + onOpenComputer(botId: string): void; +} + +function LocalVmPane({ + index, + bot, + bots, + otherBotId, + obscured, + active, + heldElsewhere, + controlPending, + onSelect, + onTake, + onRelease, + onOpenComputer, +}: LocalVmPaneProps) { + const contextId = SLOT_CONTEXTS[index]; + const botId = bot?.id ?? null; + const botName = bot?.name ?? "Local VM"; + const viewportRef = useRef(null); + const operationRef = useRef>(Promise.resolve()); + const obscuredRef = useRef(obscured); + const [retry, setRetry] = useState(0); + const [status, setStatus] = useState(null); + const [nativeState, setNativeState] = useState({ + contextId, + open: false, + status: "closed", + interactive: false, + }); + const [error, setError] = useState(null); + + useEffect(() => { + obscuredRef.current = obscured; + }, [obscured]); + + useEffect(() => { + const bridge = window.ogb?.desktopWorkspace; + return bridge?.onState((next) => { + if (next.contextId === contextId) setNativeState(next); + }); + }, [contextId]); + + useEffect(() => { + const bridge = window.ogb?.desktopWorkspace; + let alive = true; + const controller = new AbortController(); + setStatus(null); + setError(null); + setNativeState({ contextId, open: false, status: "closed", interactive: false }); + + const run = async () => { + if (bridge) await bridge.close(contextId).catch(() => {}); + if (!alive || !botId) return; + if (!bridge) { + setError("The two-desktop workspace requires the OpenMausBot desktop app."); + return; + } + try { + const raw = await api(`/api/bots/${botId}/local-computer`, { + signal: controller.signal, + }); + if (!alive) return; + const safeStatus = sanitizeLocalVmWorkspaceStatus(raw); + setStatus(safeStatus); + const viewerUrl = readyLocalVmViewerUrl(raw); + if (!safeStatus.ready || !viewerUrl) return; + + await new Promise((resolve) => requestAnimationFrame(() => resolve())); + if (!alive) return; + const bounds = elementBounds(viewportRef); + if (!bounds) throw new Error("layout-unavailable"); + const next = await bridge.open({ + contextId, + url: viewerUrl, + title: `${botName}'s Local VM`, + bounds, + }); + if (!alive) { + await bridge.close(contextId).catch(() => {}); + return; + } + setNativeState(next); + await bridge.layout([ + { contextId, bounds, visible: !obscuredRef.current && next.open }, + ]); + } catch (cause) { + if (!alive || controller.signal.aborted) return; + setError( + cause instanceof Error && cause.message === "layout-unavailable" + ? "The viewer area is not laid out yet. Retry after resizing the window." + : "OpenMausBot could not connect this Local VM viewer.", + ); + } + }; + + operationRef.current = operationRef.current.catch(() => {}).then(run); + return () => { + alive = false; + controller.abort(); + if (bridge) { + operationRef.current = operationRef.current + .catch(() => {}) + .then(() => bridge.close(contextId).then(() => undefined).catch(() => {})); + } + }; + }, [botId, botName, contextId, retry]); + + const updateLayout = useCallback(() => { + const bridge = window.ogb?.desktopWorkspace; + const bounds = elementBounds(viewportRef); + if (!bridge || !bounds || !nativeState.open) return; + void bridge + .layout([{ contextId, bounds, visible: !obscured }]) + .catch(() => setError("OpenMausBot could not position this Local VM viewer.")); + }, [contextId, nativeState.open, obscured]); + + useEffect(() => { + const element = viewportRef.current; + if (!element) return; + const observer = new ResizeObserver(updateLayout); + observer.observe(element); + window.addEventListener("resize", updateLayout); + const frame = requestAnimationFrame(updateLayout); + return () => { + cancelAnimationFrame(frame); + observer.disconnect(); + window.removeEventListener("resize", updateLayout); + }; + }, [updateLayout]); + + const label = statusLabel(status, nativeState.status); + const canDrive = Boolean(bot && status?.ready && nativeState.status === "ready" && nativeState.open); + + return ( +
+
+
+ + +
+ + {active ? "You have control" : heldElsewhere ? "Control held elsewhere" : label} +
+
+ {bot && active ? ( + + ) : bot ? ( + + ) : null} +
+ +
+
+ {!bot ? ( +
+ + Choose another bot configured for a Local VM. +
+ ) : !status && !error ? ( +
+ Checking {bot.name}'s VM… +
+ ) : status?.ready && nativeState.status !== "error" && !error ? ( +
+ Connecting live view… +
+ ) : ( +
+ +
+ {error ?? + (status?.container === "missing" + ? `${bot.name}'s Local VM has not been created.` + : status?.container === "stopped" + ? `${bot.name}'s Local VM is stopped.` + : `${bot.name}'s Local VM is not ready for a live view.`)} +
+
+ + +
+
+ )} +
+
+
+ ); +} + +export function LocalVmWorkspace({ + primaryBotId, + overlayOpen, + onClose, + onOpenComputer, +}: LocalVmWorkspaceProps) { + const { state, dispatch } = useStore(); + const eligibleBots = useMemo( + () => state.bots.filter((bot) => bot.computer === "vm" && !bot.hidden), + [state.bots], + ); + const [slots, setSlots] = useState(() => + initialLocalVmWorkspaceSlots(state.bots, primaryBotId), + ); + const slotsRef = useRef(slots); + slotsRef.current = slots; + const [controlledBotId, setControlledBotId] = useState(null); + const controlledBotIdRef = useRef(null); + const controlLeaseIdRef = useRef(null); + const controlLeaseId = controlLeaseIdRef.current ?? crypto.randomUUID(); + controlLeaseIdRef.current = controlLeaseId; + // React disables both buttons after the state update commits, but a second + // discrete event can arrive before that render. Guard the mutation itself + // so two panes can never acquire overlapping workspace leases. + const controlBusyRef = useRef(false); + const mountedRef = useRef(true); + const [controlPending, setControlPending] = useState(false); + const [controlError, setControlError] = useState(null); + const obscured = useNativeViewObscured(overlayOpen); + + const controlPort = useMemo( + () => ({ + async take(botId) { + const snapshot = await requestComputerControl(botId, "take", controlLeaseId); + dispatchControl(dispatch, botId, snapshot); + if (snapshot.held && snapshot.owned === true) controlledBotIdRef.current = botId; + else if (controlledBotIdRef.current === botId) controlledBotIdRef.current = null; + return snapshot; + }, + async release(botId) { + const snapshot = await requestComputerControl(botId, "release", controlLeaseId); + dispatchControl(dispatch, botId, snapshot); + if (controlledBotIdRef.current === botId) controlledBotIdRef.current = null; + return snapshot; + }, + async setInteractive(contextId) { + const bridge = window.ogb?.desktopWorkspace; + if (!bridge) throw new Error("The desktop workspace bridge is unavailable"); + return bridge.setInteractive(contextId); + }, + }), + [controlLeaseId, dispatch], + ); + + useEffect(() => { + setSlots((current) => { + const next = reconcileLocalVmWorkspaceSlots(current, state.bots); + return next[0] === current[0] && next[1] === current[1] ? current : next; + }); + }, [state.bots]); + + // Opening is read-only. A hold may belong to the legacy viewer or another + // surface, so observe it and surface it without silently releasing it. + useEffect(() => { + let alive = true; + const readSelected = async () => { + for (const botId of slots) { + if (!botId) continue; + try { + const snapshot = await readComputerControl(botId); + if (alive) dispatchControl(dispatch, botId, snapshot); + } catch { + // SSE can still supply the state; taking control rechecks it. + } + } + }; + void readSelected(); + return () => { + alive = false; + }; + }, [dispatch, slots]); + + useEffect(() => { + const controlled = controlledBotIdRef.current; + if (!controlled || slots.includes(controlled)) return; + void releaseLocalVmWorkspaceControl(controlPort, controlled) + .then(() => { + setControlledBotId(null); + }) + .catch(() => { + setControlError("The removed pane could not hand control back safely."); + }); + }, [controlPort, slots]); + + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + const controlled = controlledBotIdRef.current; + const bridge = window.ogb?.desktopWorkspace; + if (!bridge) { + if (controlled) bestEffortRelease(controlled, controlLeaseId); + return; + } + void bridge + .setInteractive(null) + .catch(() => {}) + .then(() => { + if (controlled) bestEffortRelease(controlled, controlLeaseId); + return bridge.close(); + }) + .catch(() => {}); + }; + }, [controlLeaseId]); + + const contextForBot = useCallback( + (botId: string) => { + const index = slotsRef.current.indexOf(botId); + return index < 0 ? null : SLOT_CONTEXTS[index]; + }, + [], + ); + + const handBack = useCallback(async () => { + const current = controlledBotIdRef.current; + if (!current) return true; + if (controlBusyRef.current) return false; + controlBusyRef.current = true; + setControlPending(true); + setControlError(null); + try { + await releaseLocalVmWorkspaceControl(controlPort, current); + setControlledBotId(null); + return true; + } catch { + setControlError("OpenMausBot could not hand control back. The workspace stayed open."); + return false; + } finally { + controlBusyRef.current = false; + setControlPending(false); + } + }, [controlPort]); + + const takeControl = useCallback( + async (botId: string) => { + if (controlBusyRef.current || controlledBotIdRef.current === botId) return; + const bridge = window.ogb?.desktopWorkspace; + const contextId = contextForBot(botId); + if (!bridge || !contextId) return; + controlBusyRef.current = true; + setControlPending(true); + setControlError(null); + try { + const alignedPort: LocalVmWorkspaceControlPort = { + ...controlPort, + async setInteractive(nextContextId) { + if (nextContextId && contextForBot(botId) !== nextContextId) { + throw new Error("The Local VM pane changed during control acquisition"); + } + return controlPort.setInteractive(nextContextId); + }, + }; + const result = await switchLocalVmWorkspaceControl( + alignedPort, + controlledBotIdRef.current, + botId, + contextId, + ); + setControlledBotId(null); + if (result.status === "held-elsewhere") { + setControlError("This VM is already controlled in another viewer. Hand it back there first."); + return; + } + if (!mountedRef.current) { + await releaseLocalVmWorkspaceControl(controlPort, botId).catch(() => {}); + return; + } + setControlledBotId(botId); + } catch { + setControlledBotId(controlledBotIdRef.current); + setControlError("Control could not switch safely. Any remaining hold stayed paused."); + } finally { + controlBusyRef.current = false; + setControlPending(false); + } + }, + [contextForBot, controlPort], + ); + + const selectSlot = useCallback( + async (index: 0 | 1, botId: string | null) => { + if (controlBusyRef.current) return; + const current = slots[index]; + if (current === botId) return; + if (current && controlledBotIdRef.current === current) { + const released = await handBack(); + if (!released) return; + } + setSlots((existing) => selectLocalVmWorkspaceSlot(existing, index, botId)); + }, + [handBack, slots], + ); + + const closeWorkspace = useCallback(async () => { + if (controlledBotIdRef.current && !(await handBack())) return; + onClose(); + }, [handBack, onClose]); + + const openComputer = useCallback( + async (botId: string) => { + if (controlledBotIdRef.current && !(await handBack())) return; + onOpenComputer(botId); + }, + [handBack, onOpenComputer], + ); + + return ( +
+
+
+ +
+
+

Local VM workspace

+

+ Two live desktops · one active controller · watch-only by default +

+
+ +
+ + {controlError && ( +
+ {controlError} +
+ )} + +
+ {([0, 1] as const).map((index) => { + const botId = slots[index]; + const bot = eligibleBots.find((candidate) => candidate.id === botId) ?? null; + return ( + void selectSlot(index, next)} + onTake={(id) => void takeControl(id)} + onRelease={() => void handBack()} + onOpenComputer={(id) => void openComputer(id)} + /> + ); + })} +
+
+ ); +} diff --git a/src/lib/local-vm-workspace.test.ts b/src/lib/local-vm-workspace.test.ts new file mode 100644 index 000000000..a71d3ebe8 --- /dev/null +++ b/src/lib/local-vm-workspace.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from "vitest"; +import { + initialLocalVmWorkspaceSlots, + nativeViewOverlayIntersects, + readyLocalVmViewerUrl, + reconcileLocalVmWorkspaceSlots, + releaseLocalVmWorkspaceControl, + sanitizeLocalVmWorkspaceStatus, + selectLocalVmWorkspaceSlot, + switchLocalVmWorkspaceControl, +} from "./local-vm-workspace"; + +const rect = (left: number, top: number, width: number, height: number) => ({ + left, + top, + width, + height, + right: left + width, + bottom: top + height, +}); + +const bots = [ + { id: "vm-a", computer: "vm" as const }, + { id: "vm-b", computer: "vm" as const }, + { id: "vm-c", computer: "vm" as const }, + { id: "cloud", computer: "cloud" as const }, + { id: "hidden", computer: "vm" as const, hidden: true }, +]; + +describe("Local VM native overlay shielding", () => { + it("hides panes only for visible intersecting overlays", () => { + const hosts = [rect(100, 100, 400, 300)]; + expect( + nativeViewOverlayIntersects(hosts, [ + { rect: rect(150, 120, 100, 80), explicit: true, visible: true, zIndex: null }, + ]), + ).toBe(true); + expect( + nativeViewOverlayIntersects(hosts, [ + { rect: rect(10, 10, 40, 40), explicit: true, visible: true, zIndex: null }, + { rect: rect(150, 120, 100, 80), explicit: false, visible: true, zIndex: 9 }, + { rect: rect(150, 120, 100, 80), explicit: true, visible: false, zIndex: 50 }, + ]), + ).toBe(false); + expect( + nativeViewOverlayIntersects(hosts, [ + { rect: rect(450, 350, 100, 100), explicit: false, visible: true, zIndex: 20 }, + ]), + ).toBe(true); + }); +}); + +describe("Local VM workspace slots", () => { + it("starts with the selected VM on the left and another eligible VM on the right", () => { + expect(initialLocalVmWorkspaceSlots(bots, "vm-b")).toEqual(["vm-b", "vm-a"]); + }); + + it("swaps a duplicate selection instead of showing one bot twice", () => { + expect(selectLocalVmWorkspaceSlot(["vm-a", "vm-b"], 0, "vm-b")).toEqual([ + "vm-b", + "vm-a", + ]); + }); + + it("removes deleted or ineligible bots and fills from remaining VM bots", () => { + expect(reconcileLocalVmWorkspaceSlots(["vm-a", "vm-b"], bots.slice(1))).toEqual([ + "vm-c", + "vm-b", + ]); + }); +}); + +describe("Local VM workspace control", () => { + function port({ held = false, owned = false } = {}) { + const calls: string[] = []; + return { + calls, + value: { + async take(botId: string) { + calls.push(`take:${botId}`); + if (held) return { held: true, helpReason: null, owned, acquired: false }; + held = true; + owned = true; + return { held: true, helpReason: null, owned: true, acquired: true }; + }, + async release(botId: string) { + calls.push(`release:${botId}`); + if (!owned) return { held, helpReason: null, released: false }; + held = false; + owned = false; + return { held: false, helpReason: null, released: true }; + }, + async setInteractive(contextId: string | null) { + calls.push(`interactive:${contextId ?? "none"}`); + return true; + }, + }, + }; + } + + it("releases and demotes the old pane before taking the next pane", async () => { + const fixture = port(); + const result = await switchLocalVmWorkspaceControl( + fixture.value, + "vm-a", + "vm-b", + "right", + ); + expect(result.status).toBe("controlled"); + expect(fixture.calls).toEqual([ + "interactive:none", + "release:vm-a", + "take:vm-b", + "interactive:right", + ]); + }); + + it("atomically observes a pane already held outside the workspace", async () => { + const fixture = port({ held: true, owned: false }); + const result = await switchLocalVmWorkspaceControl(fixture.value, null, "vm-b", "right"); + expect(result.status).toBe("held-elsewhere"); + expect(fixture.calls).toEqual(["take:vm-b"]); + }); + + it("releases only a workspace-owned current pane during close", async () => { + const fixture = port({ held: true, owned: true }); + await releaseLocalVmWorkspaceControl(fixture.value, null); + expect(fixture.calls).toEqual([]); + await releaseLocalVmWorkspaceControl(fixture.value, "vm-a"); + expect(fixture.calls).toEqual(["interactive:none", "release:vm-a"]); + }); + + it("revalidates and restores the same workspace-owned pane", async () => { + const fixture = port({ held: true, owned: true }); + const result = await switchLocalVmWorkspaceControl(fixture.value, "vm-a", "vm-a", "left"); + expect(result.status).toBe("controlled"); + expect(fixture.calls).toEqual(["take:vm-a", "interactive:left"]); + }); + + it("demotes before releasing a newly taken hold when promotion fails", async () => { + const fixture = port(); + fixture.value.setInteractive = async (contextId: string | null) => { + fixture.calls.push(`interactive:${contextId ?? "none"}`); + if (contextId === "right") throw new Error("viewer failed"); + return true; + }; + await expect( + switchLocalVmWorkspaceControl(fixture.value, null, "vm-b", "right"), + ).rejects.toThrow("viewer failed"); + expect(fixture.calls).toEqual([ + "take:vm-b", + "interactive:right", + "interactive:none", + "release:vm-b", + ]); + }); +}); + +describe("Local VM workspace status", () => { + const ready = { + mode: "per-bot", + max_instances: 2, + container: "running", + network: "loopback", + security: "hardened", + persistence: "durable", + desktopReady: true, + ready: true, + viewer_url: "http://127.0.0.1:6080/vnc.html#password=secret", + problem: "must not enter state", + }; + + it("retains only normalized readiness facts and drops URL and arbitrary text", () => { + const status = sanitizeLocalVmWorkspaceStatus(ready); + expect(status).toEqual({ + mode: "per-bot", + maxInstances: 2, + container: "running", + network: "loopback", + security: "hardened", + persistence: "durable", + desktopReady: true, + ready: true, + }); + expect(status).not.toHaveProperty("viewer_url"); + expect(status).not.toHaveProperty("problem"); + }); + + it("fails closed when any required readiness guard is unsafe", () => { + expect(sanitizeLocalVmWorkspaceStatus({ ...ready, network: "unsafe" }).ready).toBe(false); + expect(sanitizeLocalVmWorkspaceStatus({ ...ready, desktopReady: false }).ready).toBe(false); + expect(readyLocalVmViewerUrl({ ...ready, security: "unsafe" })).toBeNull(); + }); + + it("returns a ready URL only for the immediate native-view handoff", () => { + expect(readyLocalVmViewerUrl(ready)).toBe(ready.viewer_url); + }); +}); diff --git a/src/lib/local-vm-workspace.ts b/src/lib/local-vm-workspace.ts new file mode 100644 index 000000000..3235371bb --- /dev/null +++ b/src/lib/local-vm-workspace.ts @@ -0,0 +1,217 @@ +import { z } from "zod"; +import type { JsonValue } from "../../server/schema.ts"; + +export interface LocalVmWorkspaceBot { + id: string; + computer?: "cloud" | "vm" | "local" | "off"; + hidden?: boolean; +} + +export type LocalVmWorkspaceSlots = [string | null, string | null]; + +export interface LocalVmWorkspaceStatus { + mode: "shared" | "per-bot" | "unknown"; + maxInstances: number; + container: "running" | "stopped" | "missing" | "unknown"; + network: "loopback" | "unsafe" | "unknown"; + security: "hardened" | "unsafe" | "unknown"; + persistence: "durable" | "unsafe" | "unknown"; + desktopReady: boolean; + ready: boolean; +} + +export interface LocalVmWorkspaceControlSnapshot { + held: boolean; + helpReason: string | null; + /** Present only on lease-aware control mutations, never on public reads. */ + owned?: boolean; + acquired?: boolean; + released?: boolean; +} + +export interface LocalVmWorkspaceControlPort { + take(botId: string): Promise; + release(botId: string): Promise; + setInteractive(contextId: string | null): Promise; +} + +export interface NativeViewRect { + left: number; + right: number; + top: number; + bottom: number; + width: number; + height: number; +} + +export interface NativeViewOverlayCandidate { + rect: NativeViewRect; + explicit: boolean; + visible: boolean; + zIndex: number | null; +} + +/** Native views paint above renderer content. Hide them only when a visible, + * real overlay intersects a pane; ordinary positioned layout remains visible. */ +export function nativeViewOverlayIntersects( + hostRects: readonly NativeViewRect[], + candidates: readonly NativeViewOverlayCandidate[], +): boolean { + const intersects = (left: NativeViewRect, right: NativeViewRect) => + left.left < right.right && + left.right > right.left && + left.top < right.bottom && + left.bottom > right.top; + return candidates.some( + (candidate) => + candidate.visible && + candidate.rect.width > 0 && + candidate.rect.height > 0 && + (candidate.explicit || (candidate.zIndex !== null && candidate.zIndex >= 10)) && + hostRects.some((host) => intersects(candidate.rect, host)), + ); +} + +export type LocalVmWorkspaceControlResult = + | { status: "controlled"; botId: string; snapshot: LocalVmWorkspaceControlSnapshot } + | { status: "held-elsewhere"; botId: string; snapshot: LocalVmWorkspaceControlSnapshot }; + +/** Release the workspace-owned pane before inspecting or taking the next one. + * If native demotion fails, the main-process manager removes that view. */ +export async function switchLocalVmWorkspaceControl( + port: LocalVmWorkspaceControlPort, + currentBotId: string | null, + nextBotId: string, + nextContextId: string, +): Promise { + if (currentBotId && currentBotId !== nextBotId) { + await port.setInteractive(null); + await port.release(currentBotId); + } + + // The server performs this acquisition atomically. A separate read followed + // by take cannot prove ownership because another viewer may win in between. + const taken = await port.take(nextBotId); + if (!taken.held) throw new Error("The Local VM control hold was not acquired"); + if (taken.owned !== true) { + return { status: "held-elsewhere", botId: nextBotId, snapshot: taken }; + } + try { + await port.setInteractive(nextContextId); + } catch (error) { + // The native manager removes a view when demotion cannot reload it, so a + // rejected demotion is still fail-closed before the API hold is released. + await port.setInteractive(null).catch(() => {}); + await port.release(nextBotId).catch(() => {}); + throw error; + } + return { status: "controlled", botId: nextBotId, snapshot: taken }; +} + +export async function releaseLocalVmWorkspaceControl( + port: LocalVmWorkspaceControlPort, + currentBotId: string | null, +) { + if (!currentBotId) return null; + await port.setInteractive(null); + const released = await port.release(currentBotId); + return released; +} + +export function eligibleLocalVmBotIds(bots: readonly LocalVmWorkspaceBot[]): string[] { + return bots + .filter((bot) => bot.computer === "vm" && bot.hidden !== true) + .map((bot) => bot.id); +} + +export function initialLocalVmWorkspaceSlots( + bots: readonly LocalVmWorkspaceBot[], + primaryBotId: string, +): LocalVmWorkspaceSlots { + const eligible = eligibleLocalVmBotIds(bots); + const primary = eligible.includes(primaryBotId) ? primaryBotId : (eligible[0] ?? null); + return [primary, eligible.find((id) => id !== primary) ?? null]; +} + +export function selectLocalVmWorkspaceSlot( + slots: LocalVmWorkspaceSlots, + index: 0 | 1, + botId: string | null, +): LocalVmWorkspaceSlots { + const next: LocalVmWorkspaceSlots = [...slots]; + const otherIndex = index === 0 ? 1 : 0; + if (botId && next[otherIndex] === botId) next[otherIndex] = next[index]; + next[index] = botId; + return next; +} + +export function reconcileLocalVmWorkspaceSlots( + slots: LocalVmWorkspaceSlots, + bots: readonly LocalVmWorkspaceBot[], +): LocalVmWorkspaceSlots { + const eligible = eligibleLocalVmBotIds(bots); + const available = new Set(eligible); + const next: LocalVmWorkspaceSlots = [null, null]; + for (const index of [0, 1] as const) { + const id = slots[index]; + if (id && available.delete(id)) next[index] = id; + } + for (const index of [0, 1] as const) { + if (next[index]) continue; + const replacement = [...available][0]; + if (!replacement) continue; + next[index] = replacement; + available.delete(replacement); + } + return next; +} + +const localVmStatusPayloadSchema = z.object({ + mode: z.enum(["shared", "per-bot"]).optional(), + max_instances: z.number().int().positive().optional(), + container: z.enum(["running", "stopped", "missing"]).optional(), + network: z.enum(["loopback", "unsafe", "unknown"]).optional(), + security: z.enum(["hardened", "unsafe", "unknown"]).optional(), + persistence: z.enum(["durable", "unsafe", "unknown"]).optional(), + desktopReady: z.boolean().optional(), + ready: z.boolean().optional(), + viewer_url: z.string().min(1).optional(), +}); + +function parseLocalVmStatusPayload(raw: JsonValue) { + const parsed = localVmStatusPayloadSchema.safeParse(raw); + return parsed.success ? parsed.data : null; +} + +/** + * Keep only UI-safe readiness facts. The server response also contains a + * secret-bearing viewer_url and may contain arbitrary diagnostic text; neither + * is retained in React state. + */ +export function sanitizeLocalVmWorkspaceStatus(raw: JsonValue): LocalVmWorkspaceStatus { + const value = parseLocalVmStatusPayload(raw); + const mode = value?.mode ?? "unknown"; + const container = value?.container ?? "unknown"; + const network = value?.network ?? "unknown"; + const security = value?.security ?? "unknown"; + const persistence = value?.persistence ?? "unknown"; + const maxInstances = value?.max_instances ?? 0; + const desktopReady = value?.desktopReady === true; + const ready = Boolean( + value?.ready === true && + container === "running" && + network === "loopback" && + security === "hardened" && + persistence === "durable" && + desktopReady, + ); + return { mode, maxInstances, container, network, security, persistence, desktopReady, ready }; +} + +/** Return the URL only to the immediate main-process handoff. Never place it + * in component state, logs, errors, analytics or workspace state events. */ +export function readyLocalVmViewerUrl(raw: JsonValue): string | null { + const value = parseLocalVmStatusPayload(raw); + if (!value || !sanitizeLocalVmWorkspaceStatus(raw).ready) return null; + return value.viewer_url ?? null; +} diff --git a/src/types/ogb.d.ts b/src/types/ogb.d.ts index ac99786b1..0b5b8f97a 100644 --- a/src/types/ogb.d.ts +++ b/src/types/ogb.d.ts @@ -38,6 +38,21 @@ declare global { }; }; + interface DesktopWorkspaceBounds { + x: number; + y: number; + width: number; + height: number; + } + + interface DesktopWorkspaceState { + contextId: string; + open: boolean; + status: "opening" | "ready" | "error" | "closed"; + interactive: boolean; + code?: "load-failed" | "renderer-gone"; + } + interface Window { ogb?: { platform: NodeJS.Platform; @@ -88,6 +103,24 @@ declare global { open(url: string, title: string, contextId: string): Promise; onState(cb: (state: { open: boolean; contextId: string | null }) => void): () => void; }; + /** Two Local VM viewers embedded in one app window. URLs are accepted + * only by main-process validation and never return over this bridge. */ + desktopWorkspace?: { + open(input: { + contextId: string; + url: string; + title: string; + bounds: DesktopWorkspaceBounds; + }): Promise; + layout(items: Array<{ + contextId: string; + bounds: DesktopWorkspaceBounds; + visible: boolean; + }>): Promise; + setInteractive(contextId: string | null): Promise; + close(contextId?: string): Promise; + onState(cb: (state: DesktopWorkspaceState) => void): () => void; + }; /** Native folder picker; resolves null when the user cancels. */ pickFolder?(current?: string): Promise; /** Save a provider credential through Electron's OS-backed store. */ From 10efdb98818f6977a7383d291b2dadf491d648cd Mon Sep 17 00:00:00 2001 From: gus <42593099+lightcloud00@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:35:30 -0400 Subject: [PATCH 002/209] test: keep Chief capsule checks fail-closed on Windows --- server/openmaus-status-capsule.test.ts | 27 +++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/server/openmaus-status-capsule.test.ts b/server/openmaus-status-capsule.test.ts index a0138896b..4f1bbacb3 100644 --- a/server/openmaus-status-capsule.test.ts +++ b/server/openmaus-status-capsule.test.ts @@ -21,6 +21,7 @@ const OLD_SINGLE_VIEW_SHA = `sha256:${"1".repeat(64)}`; const DUAL_VIEW_SHA = `sha256:${"2".repeat(64)}`; const roots: string[] = []; const jsonObjectSchema = z.record(z.string(), z.custom()); +const posixOnly = describe.skipIf(process.getuid === undefined); interface TestCapsule extends JsonObject { schema: string; @@ -151,7 +152,7 @@ function cachePath(capsule: TestCapsule): string { return path; } -describe("readOpenMausStatus", () => { +posixOnly("readOpenMausStatus", () => { it("projects only fresh normalized two-VM capability data", () => { const capsule = successCapsule(); // Cross-language receipt produced by scripts/aos_openmausbot_status.py @@ -299,3 +300,27 @@ describe("readOpenMausStatus", () => { ).toBe("missing_or_insecure"); }); }); + +it.skipIf(process.getuid !== undefined)( + "fails closed when POSIX owner and mode checks are unavailable", + () => { + const path = cachePath(successCapsule()); + expect(readOpenMausStatus({ cachePath: path, now: NOW })).toMatchObject({ + freshness: "unknown", + reason: "missing_or_insecure", + runtimeState: "unknown", + mode: "unknown", + maxInstances: null, + readyCount: 0, + slots: [], + ui: { + twoUp: false, + maxVisible: 1, + defaultWatchOnly: false, + }, + }); + expect(openMausStatusSystemPrompt({ cachePath: path, now: NOW })).toContain( + "runtime_state=unknown", + ); + }, +); From a6855b70fc88a3a33bb3a4b857eaf914b3a151ff Mon Sep 17 00:00:00 2001 From: gus <42593099+lightcloud00@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:57:27 -0400 Subject: [PATCH 003/209] fix: harden dual VM desktop controls --- electron/desktop-workspace.cjs | 28 ++++++++---- electron/desktop-workspace.node-test.mjs | 58 ++++++++++++++++++++++++ server/openmaus-status-capsule.ts | 8 +++- src/components/ComputerPanel.tsx | 3 +- src/components/LocalVmWorkspace.tsx | 5 +- src/lib/local-vm-workspace.test.ts | 14 ++++++ src/lib/local-vm-workspace.ts | 5 +- 7 files changed, 105 insertions(+), 16 deletions(-) diff --git a/electron/desktop-workspace.cjs b/electron/desktop-workspace.cjs index 419ea03a8..314af0c8f 100644 --- a/electron/desktop-workspace.cjs +++ b/electron/desktop-workspace.cjs @@ -91,7 +91,9 @@ function createDesktopWorkspaceManager({ owner, createView, notify, partitionPre }; const removeEntry = (entry, status = "closed", code) => { - if (entries.get(entry.contextId) !== entry) return; + if (entries.get(entry.contextId) !== entry) { + return entry.terminalState ?? stateFor(entry, status, code); + } entries.delete(entry.contextId); try { entry.view.setVisible(false); @@ -104,7 +106,10 @@ function createDesktopWorkspaceManager({ owner, createView, notify, partitionPre entry.view.webContents.close({ waitForBeforeUnload: false }); } } catch {} - emit(stateFor(entry, status, code)); + const terminalState = stateFor(entry, status, code); + entry.terminalState = terminalState; + emit(terminalState); + return terminalState; }; const secureView = (entry, viewerOrigin) => { @@ -129,11 +134,11 @@ function createDesktopWorkspaceManager({ owner, createView, notify, partitionPre }; const loadMode = async (entry, interactive) => { - const current = entry.view.webContents.getURL(); - const next = desktopWorkspaceUrl(current, interactive); - entry.interactive = interactive; - emit(stateFor(entry, "opening")); try { + const current = entry.view.webContents.getURL(); + const next = desktopWorkspaceUrl(current, interactive); + entry.interactive = interactive; + emit(stateFor(entry, "opening")); await entry.view.webContents.loadURL(next.toString()); } catch { // A failed demotion must never leave an old interactive noVNC document @@ -189,8 +194,12 @@ function createDesktopWorkspaceManager({ owner, createView, notify, partitionPre removeEntry(entry, "error", "load-failed"); throw new Error("The Local VM desktop did not load"); } - if (entries.get(contextId) === entry) emit(stateFor(entry, "ready")); - return stateFor(entry, "ready"); + if (entries.get(contextId) === entry) { + const readyState = stateFor(entry, "ready"); + emit(readyState); + return readyState; + } + return entry.terminalState ?? stateFor(entry, "closed"); }, layout(items) { @@ -216,7 +225,6 @@ function createDesktopWorkspaceManager({ owner, createView, notify, partitionPre setInteractive(rawContextId) { const contextId = rawContextId == null ? null : desktopWorkspaceContextId(rawContextId); - const scopedEntries = [...entries.values()]; const targetEntry = contextId === null ? null : entries.get(contextId); if (contextId !== null && !targetEntry) { return Promise.reject(new Error("That desktop workspace slot is not open")); @@ -228,7 +236,7 @@ function createDesktopWorkspaceManager({ owner, createView, notify, partitionPre // Always finish every demotion before promoting. The queue is part of // this invariant: overlapping renderer IPC calls cannot observe a flag // change while the old interactive noVNC document is still reloading. - for (const entry of scopedEntries) { + for (const entry of entries.values()) { if ( entries.get(entry.contextId) === entry && entry.interactive && diff --git a/electron/desktop-workspace.node-test.mjs b/electron/desktop-workspace.node-test.mjs index 5a71d412c..d714288b7 100644 --- a/electron/desktop-workspace.node-test.mjs +++ b/electron/desktop-workspace.node-test.mjs @@ -124,6 +124,13 @@ test("manager keeps two isolated watch-only views and rejects duplicates or a th ); assert.equal(views.every((view) => view.webContents.url.includes("view_only=true")), true); assert.equal(views.every((view) => view.options.webPreferences.sandbox === true), true); + assert.equal(views.every((view) => view.options.webPreferences.contextIsolation === true), true); + assert.equal(views.every((view) => view.options.webPreferences.nodeIntegration === false), true); + assert.equal(views.every((view) => view.options.webPreferences.webSecurity === true), true); + assert.equal( + views.every((view) => view.options.webPreferences.allowRunningInsecureContent === false), + true, + ); assert.equal(views.every((view) => view.webContents.permissionCheck() === false), true); assert.equal(views.every((view) => view.webContents.windowOpenHandler().action === "deny"), true); assert.equal( @@ -202,6 +209,28 @@ test("manager serializes overlapping demotion and promotion calls", async () => assert.equal(views[1].webContents.url.includes("view_only=false"), true); }); +test("manager preserves one controller across reverse-order queued switches", async () => { + const { manager, open, views } = managerFixture(); + await open("left", 6080); + await open("right", 6081); + await manager.setInteractive("left"); + + let finishDemotion; + const demotionGate = new Promise((resolve) => { finishDemotion = resolve; }); + views[0].webContents.loadHook = async (url) => { + if (url.includes("view_only=true")) await demotionGate; + }; + + const switchRight = manager.setInteractive("right"); + await new Promise((resolve) => setImmediate(resolve)); + const switchBackLeft = manager.setInteractive("left"); + finishDemotion(); + await Promise.all([switchRight, switchBackLeft]); + + assert.equal(views[0].webContents.url.includes("view_only=false"), true); + assert.equal(views[1].webContents.url.includes("view_only=true"), true); +}); + test("queued interaction cannot promote a replacement pane with a reused context id", async () => { const { manager, open, views } = managerFixture(); await open("left", 6080); @@ -226,6 +255,35 @@ test("queued interaction cannot promote a replacement pane with a reused context assert.equal(views[2].webContents.url.includes("view_only=true"), true); }); +test("manager fails closed when an interactive reload derives from an invalid URL", async () => { + const { manager, open, views } = managerFixture(); + await open("left", 6080); + views[0].webContents.url = "https://desktop.example/vnc.html#password=never-print-this"; + + await assert.rejects( + manager.setInteractive("left"), + (error) => error instanceof Error && !error.message.includes("never-print-this"), + ); + assert.equal(manager.size(), 0); + assert.equal(views[0].webContents.closed, true); +}); + +test("manager does not report a pane ready after it closes during open", async () => { + const { manager, notifications, open } = managerFixture(); + const pending = open("left", 6080); + manager.close("left"); + + const state = await pending; + assert.deepEqual(state, { + contextId: "left", + open: false, + status: "closed", + interactive: false, + }); + assert.equal(notifications.at(-1)?.status, "closed"); + assert.equal(manager.size(), 0); +}); + test("manager closes panes independently and emits no viewer URL", async () => { const { children, manager, notifications, open, views } = managerFixture(); await open("left", 6080); diff --git a/server/openmaus-status-capsule.ts b/server/openmaus-status-capsule.ts index fd332040a..c6f6ad90c 100644 --- a/server/openmaus-status-capsule.ts +++ b/server/openmaus-status-capsule.ts @@ -270,7 +270,13 @@ function privateCache(path: string): Buffer | null { } catch { return null; } finally { - if (descriptor !== null) closeSync(descriptor); + if (descriptor !== null) { + try { + closeSync(descriptor); + } catch { + // Cleanup failures must not escape the fail-closed cache read. + } + } } } diff --git a/src/components/ComputerPanel.tsx b/src/components/ComputerPanel.tsx index ae6d873f1..c89b7b621 100644 --- a/src/components/ComputerPanel.tsx +++ b/src/components/ComputerPanel.tsx @@ -815,7 +815,8 @@ export function ComputerPanel({
)} - {vmStatus?.mode === "per-bot" && + {phase === "vm" && + vmStatus?.mode === "per-bot" && window.ogb?.desktopWorkspace && onOpenVmWorkspace && ( + /> + + ); + + return ( +
+ {contained ? ( +
+ {label} + {trigger} +
+ ) : ( + trigger + )} {open && (
{(() => { diff --git a/src/components/SettingsPanel.tsx b/src/components/SettingsPanel.tsx index 4fcba2965..cc78a346a 100644 --- a/src/components/SettingsPanel.tsx +++ b/src/components/SettingsPanel.tsx @@ -362,7 +362,7 @@ export function SettingsPanel({ bot }: { bot: Bot }) { return ( <> -
)}
diff --git a/src/lib/vps-computer.test.ts b/src/lib/vps-computer.test.ts new file mode 100644 index 000000000..2a2f43629 --- /dev/null +++ b/src/lib/vps-computer.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { vpsComputerNeedsReplacement, type VpsComputerStatus } from "./vps-computer"; + +const current: VpsComputerStatus = { + configured: true, + imageMatches: true, + managed: true, + container: "running", + ready: true, + problem: null, +}; + +describe("VPS computer upgrade recovery", () => { + it("offers replacement only for an existing managed container with an incompatible image", () => { + expect(vpsComputerNeedsReplacement({ ...current, imageMatches: false, ready: false })).toBe(true); + expect(vpsComputerNeedsReplacement({ ...current, container: "stopped", imageMatches: false, ready: false })).toBe(true); + expect(vpsComputerNeedsReplacement({ ...current, container: "missing", imageMatches: false, ready: false })).toBe(false); + expect(vpsComputerNeedsReplacement({ ...current, managed: false, imageMatches: false, ready: false })).toBe(false); + expect(vpsComputerNeedsReplacement(current)).toBe(false); + }); +}); diff --git a/src/lib/vps-computer.ts b/src/lib/vps-computer.ts new file mode 100644 index 000000000..a2e333b8a --- /dev/null +++ b/src/lib/vps-computer.ts @@ -0,0 +1,14 @@ +export interface VpsComputerStatus { + configured: boolean; + imageMatches: boolean; + managed: boolean; + container: "running" | "stopped" | "missing"; + ready: boolean; + problem: string | null; +} + +/** A managed container from an older release must be explicitly replaced. + * Provision deliberately refuses to overwrite it. */ +export function vpsComputerNeedsReplacement(status: VpsComputerStatus): boolean { + return status.managed && status.container !== "missing" && !status.imageMatches; +} From df32587b38fdc03d3d9de6562babac240b2d0233 Mon Sep 17 00:00:00 2001 From: hdob-macko <17006829+hdob-macko@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:06:11 +0100 Subject: [PATCH 019/209] feat(hermes): detect Nous Portal OAuth for model discovery (#409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(hermes): detect Nous Portal OAuth for model discovery HermesConfiguredModel only checked for OPENROUTER_API_KEY in ~/.hermes/.env, so a Nous Portal user logged in via OAuth (the default for `hermes setup` / `hermes login`) saw "No local models found" despite Hermes being installed, authenticated, and serving 100+ models via `hermes acp`. Now also treats the presence of ~/.hermes/config.yaml as sufficient evidence of a working provider — Hermes would not advertise models on session/new without valid credentials. This covers: - Nous Portal OAuth (no API key in .env at all) - Z.AI / other provider keys in .env - OpenRouter (the original case, still works) The `hermes acp` probe already handles the actual model discovery (fetchHermesAcpModels spawns `hermes acp`, reads session/new's availableModels). The fix is in the gate: hermesConfiguredModel returned null for Nous Portal users, so resolveModels never called fetchHermesAcpModels and the full model catalog was never read. Tests: - Updated 'commented-out key' test: null only when no config.yaml - New test: config.yaml without .env (pure Nous Portal OAuth case) - Updated blank-key tests: null only when no config.yaml - All 15 hermes.test.ts tests pass, 156 local-inject tests pass * fix(hermes): complete hosted auth detection * fix(hermes): parse provider-aware config * fix(build): bundle YAML through its ESM entry * fix(hermes): honor explicit local providers * fix(hermes): recognize named custom providers --------- Co-authored-by: hdob-macko Co-authored-by: milind-soni --- package.json | 1 + pnpm-lock.yaml | 3 + scripts/bundle-server.mjs | 14 ++++ server/drivers/acp/hermes.test.ts | 101 +++++++++++++++++++++++++-- server/drivers/acp/hermes.ts | 112 ++++++++++++++++++++++++------ 5 files changed, 204 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index bed6c8c5b..35dd481d8 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "remark-gfm": "^4.0.1", "shiki": "^4.4.3", "tailwind-merge": "^3.3.1", + "yaml": "^2.9.0", "zod": "4.4.3" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 34a60a829..9d9d4feee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: tailwind-merge: specifier: ^3.3.1 version: 3.6.0 + yaml: + specifier: ^2.9.0 + version: 2.9.0 zod: specifier: 4.4.3 version: 4.4.3 diff --git a/scripts/bundle-server.mjs b/scripts/bundle-server.mjs index eee936dfb..db58e4580 100644 --- a/scripts/bundle-server.mjs +++ b/scripts/bundle-server.mjs @@ -25,6 +25,19 @@ import { dirname, join } from "node:path"; const root = join(dirname(fileURLToPath(import.meta.url)), ".."); const server = join(root, "server"); +// yaml's Node export is CommonJS and contains dynamic requires that cannot run +// after it is inlined into our ESM-only packaged server. Its browser export is +// the same pure-JS parser without those Node shims, so resolve only this package +// to that entry while leaving every other dependency on the Node condition. +const yamlEsmPlugin = { + name: "yaml-esm", + setup(build) { + build.onResolve({ filter: /^yaml$/ }, () => ({ + path: join(root, "node_modules", "yaml", "browser", "index.js"), + })); + }, +}; + // Every file run as its own process. Keep in sync with the spawn sites above. const ENTRY_POINTS = [ "index.ts", @@ -56,6 +69,7 @@ await build({ // Written after tsc, replacing its output for these entry points. allowOverwrite: true, logLevel: "info", + plugins: [yamlEsmPlugin], }); // pi-mcp-extension.ts is NOT an OpenMausBot entry point: it is loaded by the diff --git a/server/drivers/acp/hermes.test.ts b/server/drivers/acp/hermes.test.ts index 954def3c3..02767125c 100644 --- a/server/drivers/acp/hermes.test.ts +++ b/server/drivers/acp/hermes.test.ts @@ -32,28 +32,117 @@ describe("hermesConfiguredModel", () => { }); }); - it("treats a commented-out key as not configured", () => { - // The shipped .env carries `# OPENROUTER_API_KEY=`; reading that as - // configured would offer a model that cannot authenticate. - const env = home("# OPENROUTER_API_KEY=\n", "model:\n default: anthropic/claude-opus-4.6\n"); + it.each(["GLM_API_KEY", "ZAI_API_KEY", "Z_AI_API_KEY"])( + "offers Hermes for a key-only Z.AI setup using %s", + (name) => { + const env = home(`${name}=zai-test-key\n`); + expect(hermesConfiguredModel(env)).toEqual({ + id: HERMES_CONFIG_MODEL_ID, + label: "Hermes default (config)", + custom: true, + }); + }, + ); + + it("treats a commented-out key with no config.yaml as not configured", () => { + // The shipped .env carries `# OPENROUTER_API_KEY=`; without config.yaml + // there's no evidence of a working provider, so it must not read as configured. + const env = home("# OPENROUTER_API_KEY=\n"); expect(hermesConfiguredModel(env)).toBeNull(); }); + it("treats a commented-out key with config.yaml as configured (Nous Portal)", () => { + // A Nous Portal user has OAuth tokens, not an OpenRouter API key. + // config.yaml existing is sufficient evidence of a working provider. + const env = home("# OPENROUTER_API_KEY=\n", "model:\n default: z-ai/glm-5.2\n"); + expect(hermesConfiguredModel(env)).toEqual({ + id: HERMES_CONFIG_MODEL_ID, + label: "z-ai/glm-5.2 (Hermes config)", + custom: true, + }); + }); + it.each([ "OPENROUTER_API_KEY=\n", 'OPENROUTER_API_KEY=""\n', "OPENROUTER_API_KEY='' # intentionally blank\n", "OPENROUTER_API_KEY= # configured later\n", - ])("does not treat a blank key as configured: %j", (line) => { + ])("does not treat a blank key with no config.yaml as configured: %j", (line) => { expect(hermesConfiguredModel(home(line))).toBeNull(); }); - it("returns null when there is no .env at all, leaving local-only setups unchanged", () => { + it("returns null when there is no .env and no config.yaml, leaving local-only setups unchanged", () => { const root = mkdtempSync(join(tmpdir(), "omb-hermes-bare-")); dirs.push(root); + mkdirSync(join(root, ".hermes"), { recursive: true }); expect(hermesConfiguredModel({ HERMES_HOME: join(root, ".hermes") })).toBeNull(); }); + it("offers the configured model when only config.yaml exists (Nous Portal OAuth)", () => { + // A Nous Portal user logs in via OAuth — no API key in .env, but + // config.yaml exists with a default model. This is the most common + // setup for `hermes setup` / `hermes login` users. + const root = mkdtempSync(join(tmpdir(), "omb-hermes-nous-")); + dirs.push(root); + const h = join(root, ".hermes"); + mkdirSync(h, { recursive: true }); + writeFileSync(join(h, "config.yaml"), "model:\n default: z-ai/glm-5.2\n"); + expect(hermesConfiguredModel({ HERMES_HOME: h })).toEqual({ + id: HERMES_CONFIG_MODEL_ID, + label: "z-ai/glm-5.2 (Hermes config)", + custom: true, + }); + }); + + it("does not treat an inject-only config.yaml as hosted configuration", () => { + const env = home("", "providers:\n ollama:\n base_url: http://127.0.0.1:11434/v1\n"); + expect(hermesConfiguredModel(env)).toBeNull(); + }); + + it.each(["custom", "ollama", "vllm", "llamacpp", "lmstudio"])( + "does not probe a model explicitly routed through the local %s provider", + (provider) => { + const env = home("", `model:\n default: llama3.2 # local model\n provider: ${provider}\n`); + expect(hermesConfiguredModel(env)).toBeNull(); + }, + ); + + it("keeps an explicit local provider even when a hosted key is also present", () => { + const env = home( + "OPENROUTER_API_KEY=stale-hosted-key\n", + "model:\n default: llama3.2\n provider: ollama\n", + ); + expect(hermesConfiguredModel(env)).toBeNull(); + }); + + it("keeps a named custom provider even when a hosted key is also present", () => { + const env = home( + "OPENROUTER_API_KEY=stale-hosted-key\n", + "model:\n default: local-model\n provider: custom:local\n", + ); + expect(hermesConfiguredModel(env)).toBeNull(); + }); + + it.each([ + ["scalar", "model: z-ai/glm-5.2 # selected by setup\n", "z-ai/glm-5.2"], + ["default", "model:\n default: z-ai/glm-5.2 # selected by setup\n", "z-ai/glm-5.2"], + ["model alias", "model:\n model: z-ai/glm-5.2\n", "z-ai/glm-5.2"], + ["name alias", "model:\n name: z-ai/glm-5.2\n", "z-ai/glm-5.2"], + [ + "nested default", + "model:\n provider: auto\n default:\n provider: nous\n model: z-ai/glm-5.2\n", + "z-ai/glm-5.2", + ], + ["legacy root provider", "provider: nous\nmodel:\n default: z-ai/glm-5.2\n", "z-ai/glm-5.2"], + ])("supports Hermes' %s configuration schema", (_schema, cfg, expectedModel) => { + const env = home("", cfg); + expect(hermesConfiguredModel(env)).toEqual({ + id: HERMES_CONFIG_MODEL_ID, + label: `${expectedModel} (Hermes config)`, + custom: true, + }); + }); + it("still offers the model when config.yaml is unreadable, with a generic label", () => { const env = home("OPENROUTER_API_KEY=sk-or-v1-test\n"); mkdirSync(join(env.HERMES_HOME, "config.yaml")); diff --git a/server/drivers/acp/hermes.ts b/server/drivers/acp/hermes.ts index 6da5556ad..4b72f8239 100644 --- a/server/drivers/acp/hermes.ts +++ b/server/drivers/acp/hermes.ts @@ -8,6 +8,7 @@ import { spawn } from "node:child_process"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; import type { ModelCatalog } from "../../contracts.ts"; import { decodeInjectId, hostApiKey, INJECT_SEP, localHost, mergeLocalInject } from "../local-inject.ts"; @@ -113,21 +114,78 @@ function nonEmptyDotenvValue(text: string, name: string): string | null { return raw.replace(/[ \t]+#.*$/, "").trim() || null; } -/** Model Hermes' own config will use, when a remote provider is configured. +const HERMES_HOSTED_PROVIDER_KEYS = [ + "OPENROUTER_API_KEY", + "GLM_API_KEY", + "ZAI_API_KEY", + "Z_AI_API_KEY", +] as const; + +const HERMES_LOCAL_CONFIG_PROVIDERS = new Set(["custom", "lmstudio", "ollama", "vllm", "llamacpp"]); + +function yamlString(value: unknown): string { + return typeof value === "string" ? value.trim() : ""; +} + +/** + * Read the model/provider forms accepted by Hermes' `_normalize_root_model_keys`: + * a scalar `model`, or a mapping whose id is `default`, `model`, or `name`. + * Those id fields may themselves be `{ provider, model/default }` mappings. + * An explicit outer provider wins, except `auto`, where the nested provider is + * the more specific routing choice. Root-level `provider` is Hermes' legacy + * fallback. YAML parsing also handles quotes and trailing comments correctly. + */ +function hermesConfigDefault(text: string): { model: string; provider: string } | null { + let raw: unknown; + try { + raw = parseYaml(text); + } catch { + return null; + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null; + const config = raw as Record; + const rootProvider = yamlString(config.provider); + if (typeof config.model === "string") { + const model = config.model.trim(); + return model ? { model, provider: rootProvider } : null; + } + if (!config.model || typeof config.model !== "object" || Array.isArray(config.model)) return null; + + const modelConfig = config.model as Record; + const outerProvider = yamlString(modelConfig.provider) || rootProvider; + for (const key of ["default", "model", "name"] as const) { + const candidate = modelConfig[key]; + const scalar = yamlString(candidate); + if (scalar) return { model: scalar, provider: outerProvider }; + if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) continue; + const nested = candidate as Record; + const nestedModel = yamlString(nested.model) || yamlString(nested.default); + if (!nestedModel) continue; + const nestedProvider = yamlString(nested.provider); + const provider = !outerProvider || outerProvider === "auto" ? nestedProvider || outerProvider : outerProvider; + return { model: nestedModel, provider }; + } + return null; +} + +/** Detect whether Hermes has a hosted provider configured. + * + * Hermes supports multiple auth methods: + * - OpenRouter API key in `~/.hermes/.env` (OPENROUTER_API_KEY) + * - Nous Portal OAuth (tokens stored in `~/.hermes/` — the default for + * `hermes setup` / `hermes login`) + * - Z.AI / GLM keys in `~/.hermes/.env` * - * Hermes is a BYOK harness and OpenMausBot only ever offered it *local* hosts - * (Ollama, LM Studio, EXO...). A user who has configured Hermes with a hosted - * provider — an OpenRouter key in `~/.hermes/.env`, which is how `hermes setup` - * stores it — had no selectable model at all: the picker showed "No local - * models found" and greyed the agent out, despite Hermes being installed, - * authenticated and perfectly able to answer. + * Previously only OPENROUTER_API_KEY was checked, so a Nous Portal user + * — logged in via OAuth, no OpenRouter key — saw "No local models found" + * despite Hermes being installed, authenticated, and serving 100+ models. * - * Read-only on purpose. `ensureHermesInjectProvider` writes `config.yaml`, and - * doing that from a catalog probe would rewrite the user's real Hermes config - * as a side effect of opening a menu. + * Read-only on purpose. `ensureHermesInjectProvider` writes `config.yaml`, + * and doing that from a catalog probe would rewrite the user's real Hermes + * config as a side effect of opening a menu. * - * Returns null when no hosted key is configured, which leaves the catalog - * exactly as it was for local-only setups. + * Returns null when no hosted provider is configured, which leaves the + * catalog exactly as it was for local-only setups. */ export function hermesConfiguredModel( env: Record = process.env, @@ -137,20 +195,32 @@ export function hermesConfiguredModel( try { secrets = readFileSync(join(dir, ".env"), "utf8"); } catch { - return null; + /* .env may not exist — check OAuth below */ } - // Only an uncommented, non-empty assignment counts; the shipped file has the - // key present but commented out, and that must not read as "configured". - if (!nonEmptyDotenvValue(secrets, "OPENROUTER_API_KEY")) return null; - let model = ""; + const hasHostedProviderKey = HERMES_HOSTED_PROVIDER_KEYS.some((name) => nonEmptyDotenvValue(secrets, name)); + + // `hermes login` / `hermes setup` records the selected default in + // config.yaml while the OAuth token lives in Hermes' auth store. An explicit + // local/custom provider must not trigger the hosted catalog probe. + let configuredDefault: { model: string; provider: string } | null = null; try { - const cfg = readFileSync(join(dir, "config.yaml"), "utf8"); - const m = /^[ \t]*default[ \t]*:[ \t]*["']?([\w./:+-]+)["']?[ \t]*$/m.exec(cfg); - if (m) model = m[1]; + configuredDefault = hermesConfigDefault(readFileSync(join(dir, "config.yaml"), "utf8")); } catch { - /* config unreadable — the id still works, only the label is less specific */ + /* config may not exist or may be unreadable */ } + + const configuredProvider = configuredDefault?.provider.toLowerCase() ?? ""; + // The model/provider selected in config.yaml is the user's explicit routing + // choice. A stale hosted key must not override an explicitly local setup. + const configIsLocal = + HERMES_LOCAL_CONFIG_PROVIDERS.has(configuredProvider) || configuredProvider.startsWith("custom:"); + if (configuredDefault && configIsLocal) return null; + + const configIsHosted = configuredDefault !== null; + if (!hasHostedProviderKey && !configIsHosted) return null; + + const model = configuredDefault?.model ?? ""; // `custom: true` is not cosmetic. ModelPicker renders a custom-only agent's // *custom* pane exclusively, and that pane lists only options carrying this // flag; anything without it lands in the "official" bucket the pane never From c2dc783f48e96319aacd57d668cf0fcdee63b9ac Mon Sep 17 00:00:00 2001 From: Omkar Satpute Date: Mon, 24 Aug 2026 22:43:57 +0530 Subject: [PATCH 020/209] feat(composer): attach a file, and switch auto mode where you type (#395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(composer): attach a file, and switch auto mode where you type Auto mode already existed, as a toggle most of the way down a bot's settings panel. Nobody found it: the moment you want it is the moment a bot stops to ask, and that moment is spent looking at the composer, not at settings. It now sits in the composer as an Auto pill, switching the same bot.autoApprove, behind the same acknowledgement dialog a bot that drives this computer has always required. A room has no pill — auto mode belongs to one bot, and a room has several. What it does NOT do is widen the grant. Anything matching a destructive pattern or reaching for a secret still cards, exactly as before; the pill is a shorter route to the switch, not a new switch. Attaching a file was drop-or-paste only, which is invisible until someone tells you about it. There is now a + button. It opens a picker and hands the files to the same intake the drop path uses, so a picked file and a dropped one cannot sort differently — an image uploads, a file on disk becomes a path, and anything with neither is named in the notice rather than dropped in silence. That intake was previously inline in the drop effect; it is now a tested function both paths call. Co-Authored-By: Claude Opus 5 (1M context) * fix(composer): surface attachment intake failures --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: milind-soni --- src/components/Composer.tsx | 97 +++++++++++++++++++++++++- src/components/ComposerAttachments.tsx | 48 +++++-------- src/lib/composer-attachments.ts | 43 ++++++++++++ src/lib/intake-files.test.ts | 70 +++++++++++++++++++ 4 files changed, 226 insertions(+), 32 deletions(-) create mode 100644 src/lib/intake-files.test.ts diff --git a/src/components/Composer.tsx b/src/components/Composer.tsx index fcbaf38af..33b1326a7 100644 --- a/src/components/Composer.tsx +++ b/src/components/Composer.tsx @@ -1,14 +1,16 @@ import { track } from "@/lib/analytics"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { ArrowUp, Clock, Mic, Square, Users, X } from "lucide-react"; +import { ArrowUp, Clock, Mic, Plus, Square, Users, X, Zap } from "lucide-react"; import { useStore, visibleMessages, type Bot, type Group } from "@/state/store"; import { cn } from "@/lib/cn"; import { useComposerDraft } from "@/lib/drafts"; import { MausAvatar } from "./Avatar"; -import { ComposerAttachments } from "./ComposerAttachments"; +import { ComposerAttachments, pathForFile } from "./ComposerAttachments"; +import { LocalComputerAutoWarning } from "./LocalComputerAutoWarning"; import { composeMessage, imageAttachmentFromFile, + intakeFiles, isImageFile, isLongPaste, pasteAttachment, @@ -163,6 +165,36 @@ export function Composer({ ? state.pendingQueued?.[bot.threadId]?.map((entry) => entry.text).join("\n") : undefined; // a chip on its own is a message: the send control has to appear for it + const fileInput = useRef(null); + const [autoWarn, setAutoWarn] = useState(false); + const [attachmentNotice, setAttachmentNotice] = useState(null); + // Auto mode belongs to one bot; a room has several, each with its own. + const autoBot = group ? undefined : bot; + const pickFiles = async (picked: FileList | null) => { + if (!picked?.length) return; + const { attachments: added, notice } = await intakeFiles(Array.from(picked), { + allowImages: engineSupportsImages, + getPath: pathForFile, + uploadImage: imageAttachmentFromFile, + }); + if (added.length) addAttachments(added); + // Keep file-specific failures beside the attachments. A successful + // overlapping intake must not erase an earlier failure before it is read. + if (notice) setAttachmentNotice(notice); + }; + const toggleAuto = () => { + if (!autoBot) return; + // Turning it on for a bot that drives THIS computer is the one case that + // has to be acknowledged first. The flag the dialog sends is stripped by + // the reducer rather than stored, so — exactly like the settings panel — + // the warning is shown on every switch-on, not just the first. + if (!autoBot.autoApprove && autoBot.computer === "local") { + setAutoWarn(true); + return; + } + dispatch({ type: "updateBot", botId: autoBot.id, patch: { autoApprove: !autoBot.autoApprove } }); + }; + const hasContent = Boolean(text.trim()) || attachments.length > 0; const send = () => { if (locked) return; @@ -326,8 +358,31 @@ export function Composer({ onAdd={addAttachments} onRemove={removeAttachment} allowImages={engineSupportsImages} + notice={attachmentNotice} + onNotice={setAttachmentNotice} /> -
+
+ { + void pickFiles(e.target.files); + // same file twice in a row still fires onChange + e.target.value = ""; + }} + /> + {!locked && ( + + )}