From 25560b16cad91b065bb0f69e1e2c1965cb6a11c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= Date: Tue, 6 Oct 2026 20:55:54 +0200 Subject: [PATCH 1/4] Environment browser: the people's view, one tab of a person for one remote tab (F39, DEV-6646) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the pilot Environment the Organization Admin found agent-browser's dashboard, the view of F38, "velmi zabugované". It is a browser inside a browser, its second token broke every open view on each restart, its links were pinned to stream ports, paste never reached the page, and a passkey request opened Chrome's native window outside the page, which blocked the page until it was closed on the screen. Root decision 0191 points 11–18 set the model: a person's tab is a window into exactly one remote tab, agent and person work in it together, and nothing outside the page may block it. The view is now the Platform's own thin service over the Chrome DevTools protocol, `lazurio browser serve`, run by lazurio-browser-view.service: - `/` opens a new remote tab (a new window of the default context, at the person's size) and becomes `/t/`, the view of exactly that tab; the bar has back, forward, reload and the address, and the title and icon are read from the page; - a pop-up that can reach its opener shows over the page in the same person's tab; a new tab is offered with one click and posted to an embedding app; - keys keep the person's codes, ⌘ shortcuts become Ctrl, paste inserts the person's clipboard and copy takes the remote selection, read in an isolated world without enabling the Runtime domain; - JavaScript dialogs and file choosers are answered in the view; a file chooser with nobody watching is dropped; - the window takes the last acting person's size, so `lazurio browser window` pins a thread's session without a fixed viewport; - a tab the view opened closes 30 s after its last viewer, unless an agent-browser session is bound to it; - the gateway is the only admission: the service checks Host on every request and Origin on every socket and change, frames only for the Environment's own origins and listens on loopback. The browser unit gains `--deny-permission-prompts` and the Lazurio extension, written next to the profile: it moves a page's new tab into a window of its own and, until the Environment's Bitwarden holds its passkeys, declines WebAuthn at once. A changed browser unit now restarts the browser on install and update (F38 point 1 changes); the profile keeps its sign-ins. `/.lazurio/browser.json` answers the view of a thread's tab, so T3 Code's Browser panel needs no change; without a session it answers a new tab and the view of every window is retired. The Folder manual (base-instructions-31) tells agents to send their tab's link, to work together with the Operator and that passkeys are declined for now. Doctor asks the service's health. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/decisions.md | 174 +++ docs/environment-tools.md | 2 +- docs/hosted-entry.md | 15 +- docs/launchpad-development.md | 11 +- docs/machine-handover.md | 3 +- docs/update.md | 24 +- scripts/preview-launchpad.ts | 21 +- src/browser/cli.ts | 37 +- src/browser/people/bundle.ts | 26 + src/browser/people/cdp.ts | 173 +++ src/browser/people/client.ts | 1331 +++++++++++++++++ src/browser/people/extension.ts | 132 ++ src/browser/people/hub.ts | 1232 +++++++++++++++ src/browser/people/input-map.ts | 355 +++++ src/browser/people/page.ts | 169 +++ src/browser/people/protocol.ts | 321 ++++ src/browser/people/serve.ts | 72 + src/browser/people/service.ts | 300 ++++ src/browser/units.ts | 122 +- src/browser/view.ts | 180 +-- src/browser/window.ts | 9 +- src/cli.ts | 10 +- src/folder/manual.ts | 34 +- src/folder/render.ts | 2 +- src/launchpad/browser-panel-view.ts | 30 +- src/launchpad/server.ts | 15 +- .../__snapshots__/folder-manual.test.ts.snap | 56 +- .../__snapshots__/folder-render.test.ts.snap | 28 +- tests/browser-panel-view.test.ts | 44 +- tests/environment-browser-view-client.test.ts | 709 +++++++++ tests/environment-browser-view.test.ts | 1013 +++++++++++++ tests/environment-browser.test.ts | 239 +-- tests/folder-template-revision.test.ts | 4 +- tests/folder-tools.test.ts | 4 +- tests/update-install.test.ts | 5 +- 35 files changed, 6374 insertions(+), 528 deletions(-) create mode 100644 src/browser/people/bundle.ts create mode 100644 src/browser/people/cdp.ts create mode 100644 src/browser/people/client.ts create mode 100644 src/browser/people/extension.ts create mode 100644 src/browser/people/hub.ts create mode 100644 src/browser/people/input-map.ts create mode 100644 src/browser/people/page.ts create mode 100644 src/browser/people/protocol.ts create mode 100644 src/browser/people/serve.ts create mode 100644 src/browser/people/service.ts create mode 100644 tests/environment-browser-view-client.test.ts create mode 100644 tests/environment-browser-view.test.ts diff --git a/docs/decisions.md b/docs/decisions.md index 28a0a8e6a..f004893ab 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -5116,3 +5116,177 @@ view. Its input sends punctuation with ASCII codes as Windows virtual-key codes, arrives as Delete (vercel-labs/agent-browser#1380, fix #1382 open), and its sessions, activity and console columns are too much for people. A view in the shape of the right-panel browser of ChatGPT/Codex Desktop is the second phase (DEV-6646). + +## F39 — The people's view of the Environment browser: one tab of a person is one remote tab + +**Decided by Matěj 2026-10-06 after the pilot (root decision 0191 points 11–18, plan +DEV-6646); implemented in this revision.** On the pilot Environment the view of F38 +point 5, agent-browser's dashboard, was "velmi zabugované" for the Organization Admin: +- it is a browser inside a browser, with its own sessions and tabs; +- its second token broke every open view on each restart (#234); +- its links were pinned to stream ports that change when a session restarts; +- paste never reached the page; +- a passkey request opened Chrome's native window outside the page, and the page took + no input until someone closed that window on the screen. + +The Admin's model replaces it. A person's tab in any app (web T3 Code, ChatGPT/Codex +Desktop's right panel, a browser, a phone) is a window into exactly one tab of the +Environment browser. The view is the Platform's own thin service over the Chrome +DevTools protocol, and agent-browser stays the agents' tool. This decision changes +F38 points 1, 5 and 6 as stated below; points 2–4 and 7 stand. + +1. **Addresses.** The view keeps the gateway's `browser.` origin and loopback port of + F38 (`entry.browser`). + - `GET /` is a page that opens a new remote tab (`POST /api/tabs`) and replaces its + own address with `/t/`, where `` is the tab's DevTools target id. + Creation is a POST, so nothing that only fetches `/` opens a tab. + - `GET /t/` is the view of that one tab, and `/t//live` is its WebSocket. + - The view page has only a thin bar (back, forward, reload, the address) and takes + its title and icon from the remote page. It has no list of sessions or tabs. + Chrome reports a target's title only as its address, so the service reads the + page's title and icon itself, at each load and every 2 s while someone views + the tab. +2. **Each remote tab is its own window** (0191 point 11), because Chrome paints only + visible windows. A tab the view opens is a new window of the default context (CDP + `Target.createTarget` with `newWindow`, the sign-ins of F38 point 2). A tab a page + opens in an existing window is moved into a window of its own by the Lazurio + extension (point 7), which keeps its opener. +3. **The window takes the person's size.** When a person opens or resizes the view, + the window of that tab is resized to the view's page area, clamped to the screen, + so the page lays out for the person. With two people on one tab, the last one who + acted wins. `lazurio browser window` therefore no longer fixes a thread's viewport + (F38 point 3 did, for the dashboard's pointer mapping); it pins the session with + `get url`. An agent's own `set viewport` still overrides the page size, and the + view then shows the frame scaled. +4. **Pop-ups and new tabs** (0191 point 12). A page target whose opener is the viewed + tab: + - **with access to its opener** (`canAccessOpener`: `window.open` with a reference, + the shape of "Sign in with Google") is a pop-up. It shows over the page in the + same person's tab, and disappears when it closes; + - **without access** (`target=_blank`, `noopener`) is a new tab. The view offers + it with one click ("Stránka otevřela novou záložku — Otevřít") and posts + `lazurio-browser:new-tab` with its `/t/` address to the embedding app. Web + T3 Code opens such a tab as a tab of its Browser panel in its own revision + (Lazurio/t3code); until then it gets the same one-click bar. +5. **Co-control** (0191 point 13). Agent and person click and type in the same tab at + once. The view never locks out the agent and offers no "take over". +6. **Input** (0191 point 15). + - **Keys** are sent with the person's `key` and `code` and the Windows virtual-key + code the person's browser reports, so `.`, `-` and `'` arrive unchanged + (Lazurio/LazurioPlatform#218). + - **macOS shortcuts** (`⌘` with a letter) become `Ctrl` on the Linux browser. + - **Paste** (`⌘V`/`Ctrl+V`) inserts the text of the person's clipboard + (`Input.insertText`). + - **Copy and cut** put the remote selection into the person's clipboard. The + service reads the selection in an isolated world when it may have changed + (a released mouse button or key) and when the person presses `Ctrl` or `⌘`, + just before a copy. It never enables the Runtime domain: a binding would + need it, and pages can detect it as automation during a person's sign-in. + - **Touch** is sent as touch, and a tap on a phone opens its keyboard. + - The view uses the `paste` and `copy` events, so it needs no clipboard + permission even inside an app's frame. +7. **Nothing outside the page blocks the view** (0191 point 16 and 17). + - The browser starts with `--deny-permission-prompts` and the Lazurio extension + (`--load-extension`, written by `install` and `update` next to the profile, never + into it). The extension: + - moves a new tab of a shared window into a window of its own; + - until the Environment's Bitwarden is installed (0191 point 17, + Lazurio/LazurioPlatform#204), answers every WebAuthn request of a page + (`navigator.credentials` with `publicKey`) with `NotAllowedError` at once, and + reports no platform authenticator and no conditional mediation. Chrome's + native passkey window never opens, and sites offer another way. + - The view server keeps a DevTools session on every page: + - **JavaScript dialogs** (alert, confirm, prompt, beforeunload) show in the view + and are answered from it; + - **file choosers** are intercepted on every page. In the view, the person picks + files, which are uploaded to the Environment and set on the input. With no + person watching, the chooser is dropped rather than shown on the screen. + - HTTP authentication and the print preview are not handled in this revision (see + Not decided). +8. **One admission: the gateway** (0191 point 14). + - The view has no token of its own. + - Every request must name the view's own host. Every WebSocket and every request + that changes something must carry the view's own `Origin`. `lazurio.io` is not a + public suffix, so another `*.lazurio.io` app is same-site, and the Origin check + is the barrier. + - Pages may frame the view only from the Environment's own origins + (`frame-ancestors 'self' https://*.`). + - The service listens on loopback only. An agent of the Environment already reaches + the browser's DevTools, so loopback adds no reach. +9. **A closed person's tab closes its remote tab** (0191 point 11) after a grace of 30 + seconds without any viewer. This applies only to a tab the view itself opened, and + only when no agent-browser session is bound to it (`/.target`, + as F38 point 3 reads it). A view that is opened again within the grace keeps the + tab. After a restart of the service the record is gone, and nothing is closed. +10. **The service replaces the dashboard unit.** + - `lazurio-browser-view.service` runs ` browser serve --port + --origin ` as a long-running service (`Type=simple`, + `Restart=always`). + - `lazurio-browser.service` gains `--deny-permission-prompts` and + `--load-extension`. + - A changed text of the browser unit now restarts the browser on `install` and + `update`, which changes F38 point 1. That restart closes every window: agents + reopen theirs with `lazurio browser window`, and the profile's sign-ins stay. + Without it, the extension would wait for an unrelated restart. + - The screen is still never restarted. +11. **The links.** + - `lazurio browser window` prints the thread's own view, `https://browser.<…>/t/`. + - `GET /.lazurio/browser.json?session=` answers that address for a thread's + window: T3 Code's Browser panel frames it unchanged. + - Without a session it answers the view's `/`, so the Launchpad's panel opens a + new remote tab. The view of all windows is retired (0191 point 18); only the + desktop of the second wave shows them all. + - `/.lazurio/browser` redirects the same way. + - No address carries a token any more. +12. **Folder manual.** The agent sends its tab's link. It works together with the + Operator in the same tab. At a sign-in, 2FA or CAPTCHA it asks the Operator to + finish it there and does not type into the same field meanwhile (template revision + `base-instructions-31`). +13. **Doctor** asks the service's `GET /.lazurio/health` on loopback instead of the + dashboard's page, which also reports whether the service reaches the browser. + +**Failure modes.** + +| Failure | What happens | +| --- | --- | +| The browser restarts (crash, update) | The service reconnects every 2 s; open views show "reconnecting" and resume on their tab if it survived, else "this tab no longer exists" with "open a new tab" | +| The tab is closed by an agent or a page | Its views say so and offer a new tab | +| The person's network drops | The view reconnects with backoff; frames are not queued, the next frame is current | +| A slow viewer | Frames are acknowledged to Chrome only when every viewer's socket has drained below 1 MiB, so Chrome slows down instead of the service buffering | +| Two people and an agent type at once | Accepted (co-control); keys interleave as they would on one keyboard | +| A request from another `*.lazurio.io` page | Refused by the Origin check (403) | +| A page answers a passkey request before the extension runs | The content script runs at `document_start` in every frame, before page scripts | + +**Variants considered.** + +| Alternative | Disposition | +| --- | --- | +| Keep agent-browser's dashboard and patch it (#1618 for paste) | Its model (sessions and tabs inside one view) is the one the Admin rejected; rejected | +| noVNC, Selkies or neko for the view | They show the whole screen or the whole browser, not one tab; the desktop of the second wave may use one of them; rejected for the view | +| browserless or Steel | Separate browser platforms instead of the Environment's one Chromium; rejected | +| Tabs of one window with a tab strip in the view | Background tabs do not paint, and the Admin rejected switching tabs inside one person's tab; rejected | +| A token of the view (as the dashboard) | A second admission that broke on restarts; the gateway is the admission; rejected | +| Chrome policy to block WebAuthn | No policy turns WebAuthn off for every site without its native UI; an extension answers before the page; selected | +| A thin view service over CDP, one tab per person's tab, an extension for windows and passkeys (selected) | The smallest own piece that gives the Admin's model; everything else stays Chrome, agent-browser and the gateway | + +**Not decided here.** The Bitwarden extension and passkeys through it (#204), HTTP +authentication and print preview in the view, T3 Code's automatic panel tab for a +page's new tab (Lazurio/t3code), the desktop and Computer Use (second wave) and +MausBot's move to this browser (third wave). + +Verified by: +- `tests/environment-browser-view.test.ts` against a scripted DevTools peer: routes, the Host and Origin checks, tab creation, pop-up and new-tab classification, the grace close, dialogs, file choosers, the title read from the page, the selection read on demand, frame flow control, reconnecting, and the extension's two scripts; +- `tests/environment-browser-view-client.test.ts`: input mapping, frames and the protocol; +- `tests/environment-browser.test.ts`: units, convergence, the browser restart on a changed unit, doctor and links. + +Before the release, the service ran against a real headless Chrome for Testing 155 with the extension, viewed in a browser: +- a new tab from `/`, navigation from the bar, and the page's title; +- typing `j.-'@`, a paste and a copy; +- a sign-in pop-up shown in the same tab, which closed itself and reached its opener; +- the offer of a new tab; +- a prompt answered in the view; +- a file uploaded into the page's input; +- a passkey request declined at once (`NotAllowedError`); +- the window taking the view's size. + +Branded Google Chrome ignores `--load-extension`; Chrome for Testing, which the Environment runs, loads it. The pilot Environment's browser check (decision 0178) follows the release. diff --git a/docs/environment-tools.md b/docs/environment-tools.md index a16113df7..511d25c0a 100644 --- a/docs/environment-tools.md +++ b/docs/environment-tools.md @@ -457,7 +457,7 @@ The unit text is in [product update](update.md#state-on-disk). | Git and GitHub CLI (`gh`) | Diagnose availability and required capabilities; propose explicit preparation of missing tools | Authorize package/system changes; authenticate as the intended Operator and grant actual repo access | | Module runtime (for example Bun) | Coordinate the module's declared preparation; report missing/incompatible runtime without claiming readiness | Module owns exact dependency/runtime requirements and preparation; operator approves installation | | Codex / Claude harness | Diagnose the selected harness, instruction loading and required capabilities; provide one Folder-owned instruction contract; in a Remote Environment start the operator's Codex app-server daemon at boot (`lazurio-codex-app-server.service`, F29) and report it in `lazurio doctor` | Choose the harness, accounts/model access and consent; complete provider-native sign-in; install, update and configure Codex itself | -| Environment browser (Remote Environment, F38) | Run the virtual screen, the one Chromium with its persistent profile and agent-browser's dashboard as user units; open a thread's window (`lazurio browser window`); hand the view over behind the gateway; report it in `lazurio doctor` | Machines delivers Xvfb, Chrome for Testing through agent-browser and the `browser.` route with `entry.browser`; the person signs in to websites in the view and decides what may be signed in there | +| Environment browser (Remote Environment, F38, F39) | Run the virtual screen, the one Chromium with its persistent profile and the Lazurio extension, and the people's view (`lazurio browser serve`) as user units; open a thread's window (`lazurio browser window`); answer the view of a thread's tab behind the gateway; report it in `lazurio doctor` | Machines delivers Xvfb, Chrome for Testing through agent-browser and the `browser.` route with `entry.browser`; the person signs in to websites in the view and decides what may be signed in there | | Credentials | Use an existing approved provider/credential interface; retain only non-secret diagnostic outcomes | Existing credential owner retains custody, rotation and revocation | The table describes a private workspace or local Machine, where one person is the diff --git a/docs/hosted-entry.md b/docs/hosted-entry.md index f2564a135..e88c557df 100644 --- a/docs/hosted-entry.md +++ b/docs/hosted-entry.md @@ -210,13 +210,16 @@ Lazurio MausBot is entered the same way when the entry records it ([contract](launchpad-development.md#lazurio-mausbot-entry)): `mausbotOrigin` is in the public parts, the loopback port is not, and the code rides only in the fragment of `/pair`. -The Environment browser's view ([F38](decisions.md#f38--the-environment-browser-of-a-remote-environment-one-chromium-a-window-per-thread-a-view-behind-the-gateway)) +The Environment browser's people's view ([F38](decisions.md#f38--the-environment-browser-of-a-remote-environment-one-chromium-a-window-per-thread-a-view-behind-the-gateway), +[F39](decisions.md#f39--the-peoples-view-of-the-environment-browser-one-tab-of-a-person-is-one-remote-tab)) follows the same pattern: `browserOrigin` is in the public parts, the loopback port is -not; `GET /.lazurio/browser.json` and the hand-over `GET /.lazurio/browser` answer the -view's address with the dashboard's access token only in the fragment, behind this -admission and on the forks' origins too (the gateway forwards `/.lazurio/*` to the -Launchpad). Asked for a session that has no window yet, they open that session's window -first, so a T3 thread's panel shows the thread's own window (F38, addendum 2026-10-06). +not. `GET /.lazurio/browser.json` and the hand-over `GET /.lazurio/browser` answer, behind +this admission and on the forks' origins too (the gateway forwards `/.lazurio/*` to the +Launchpad), the view's address of a session's tab, `/t/`, or +`/` for a new tab without a session. No address carries a token: the +view's only admission is the gateway's. Asked for a session that has no window yet, +they open that session's window first, so a T3 thread's panel shows the thread's own tab +(F38, addendum 2026-10-06). ### Files links (decision F35) diff --git a/docs/launchpad-development.md b/docs/launchpad-development.md index 674c17c05..167c9197b 100644 --- a/docs/launchpad-development.md +++ b/docs/launchpad-development.md @@ -1769,9 +1769,10 @@ builds no viewer of its own. new tab" and Close; Escape inside it closes it. The first-run tour steps aside while it is open. - **What it shows** (`browser-panel-view.ts`, pure; `browser-panel.ts` draws it). - Opening and Reload ask `GET /.lazurio/browser.json` on this origin without a session - (every window): `credentials: "same-origin"`, no cache, no redirect followed, given - up after 20 s. The view is embedded only from `{available: true, view}` whose `view` + Opening and Reload ask `GET /.lazurio/browser.json` on this origin without a session, + which answers the people's view of a new remote tab (F39; the view of every window is + retired, root decision 0191 point 18): `credentials: "same-origin"`, no cache, no + redirect followed, given up after 20 s. The view is embedded only from `{available: true, view}` whose `view` is an https URL on exactly the recorded `browserOrigin`, without credentials, as written and never on the page's own origin, in a frame with `allow="clipboard-read; clipboard-write; fullscreen"`, `referrerpolicy="no-referrer"` and the sandbox of T3 @@ -1782,9 +1783,7 @@ builds no viewer of its own. a timeout, another shape) says to open it in a new tab, where the gateway's sign-in can show, which a frame cannot. - **"Open in a new tab"** leads to the view once it is known and to the hand-over - `/.lazurio/browser` otherwise. The view's address carries the dashboard's token in - its fragment: it lives only in the frame and that link while the panel is open, and - never in `localStorage`, `sessionStorage` or the Folder. + `/.lazurio/browser` otherwise. The view's address carries no token (F39 point 8). - **Verified** by `tests/browser-panel-view.test.ts` (where it is offered, from the projected handovers with and without the route; the view accepted from the server's own resolution and refused in every other shape; the read's request and its failures; diff --git a/docs/machine-handover.md b/docs/machine-handover.md index 68483a628..bba4d533a 100644 --- a/docs/machine-handover.md +++ b/docs/machine-handover.md @@ -357,7 +357,8 @@ Launchpad shows nothing for it. `browser` is optional and present only on a Machine whose gateway roster routes the Environment browser's view (root decision 0191, [F38](decisions.md#f38--the-environment-browser-of-a-remote-environment-one-chromium-a-window-per-thread-a-view-behind-the-gateway)): -the view's origin and the loopback port its dashboard listens on. It is also the +the view's origin and the loopback port the people's view service listens on (F39). +It is also the signal the installer converges the browser's units on. Absent, the Environment has no Environment browser. diff --git a/docs/update.md b/docs/update.md index d8f057996..d29a01366 100644 --- a/docs/update.md +++ b/docs/update.md @@ -586,19 +586,25 @@ Codex unit: - `lazurio-browser.service`: the newest Chrome for Testing agent-browser installed under `~/.agent-browser/browsers/` (the path Machines' AppArmor profile admits), `exec`'d by `/bin/sh` so Chrome is the unit's main process, headed on `DISPLAY=:1`, - profile `~/.local/share/lazurio-browser/profile`, DevTools on loopback port 9222; + profile `~/.local/share/lazurio-browser/profile`, DevTools on loopback port 9222, + `--deny-permission-prompts` and the Lazurio extension + (`--load-extension=~/.local/share/lazurio-browser/extension`, [F39](decisions.md#f39--the-peoples-view-of-the-environment-browser-one-tab-of-a-person-is-one-remote-tab) + point 7) with `LAZURIO_BROWSER_EXTENSION=` in the unit's environment; `BindsTo=`/`After=` the display, `Restart=always`, and `RestartPreventExitStatus=78` for "no Chrome to run". -- `lazurio-browser-view.service`: `agent-browser dashboard start --port --allowed-origins `, oneshot with - `RemainAfterExit=yes`, `KillMode=process`, `ExecStop=` `dashboard stop` and - `AGENT_BROWSER_CDP=9222`, so what the dashboard runs uses the Environment browser. +- `lazurio-browser-view.service`: the people's view (F39), this base's ` + browser serve --port --origin + `, a long-running service with `Restart=always`. The screen's and the browser's texts name no base, Folder or address; the view's names -the entry's port and origin. Each is written when its text differs (then one -`daemon-reload`), all three are enabled and started; `install` and `update` never stop -or restart the screen or the browser (that would close every agent's window), and -restart the view only when its own text changed. The result carries +this base's selector and the entry's port and origin. `install` and `update` first +write the extension's files that differ next to the profile, never into it. Each unit +is written when its text differs (then one `daemon-reload`), and all three are enabled +and started. `install` and `update` never stop or restart the screen. They restart the +browser only when its own text changed: a new flag or a new extension, whose digest is +in the text. That restart closes every window; agents open theirs again, and the +profile keeps its sign-ins (F39 point 10). They restart the view when its own text +changed. The result carries `environmentBrowser`: `{"state":"enabled"}`, `{"state":"skipped-not-hosted"}`, `{"state":"skipped-not-declared"}` (no `entry.browser`; nothing is written or stopped), `{"state":"foreign-unit","unit","next"}` or `{"state":"failed","step","next"}`; never a diff --git a/scripts/preview-launchpad.ts b/scripts/preview-launchpad.ts index f67bd8a2a..c0ab2a73c 100644 --- a/scripts/preview-launchpad.ts +++ b/scripts/preview-launchpad.ts @@ -246,24 +246,9 @@ const app = await startLaunchpad( { home, platform: process.platform, folder }, undefined, undefined, - // The Environment browser's view (decision F38): agent-browser's dashboard - // answers its address with a synthetic token and no session runs; nothing - // is started. - { - run: async () => ({ - exitCode: 0, - stdout: JSON.stringify({ - success: true, - data: { - access_urls: [ - `${browserOrigin}/#dashboard-access-token=${"0".repeat(64)}`, - ], - }, - }), - }), - fetch: async () => Response.json([]), - env: { HOME: home }, - }, + // The Environment browser's people's view (decision F39): a thread's tab + // answers a synthetic target id; nothing is opened. + { openWindow: async () => ({ targetId: "0".repeat(32) }) }, ); let proxy: string | null = null; if (mode !== "local") { diff --git a/src/browser/cli.ts b/src/browser/cli.ts index 175bf597f..825a52fcf 100644 --- a/src/browser/cli.ts +++ b/src/browser/cli.ts @@ -17,7 +17,7 @@ import { } from "../update/errors"; import { runProcess } from "../update/self-check"; import { browserCdpPort } from "./units"; -import { browserEntryOf } from "./view"; +import { browserEntryOf, browserViewUrl } from "./view"; import { BrowserWindowFailure, cdpSeams, @@ -27,21 +27,25 @@ import { } from "./window"; /** `lazurio browser`: the agent's way into the Environment browser of a - * Remote Environment (decision F38). */ + * Remote Environment (decisions F38 and F39). */ export const browserHelp = `browser window [--session ] [--url ] [--folder ] [--json] The calling thread's own window of the Environment browser: the one its agent-browser session is bound to while it is open, otherwise a new window of the shared browser (with the Environment's sign-ins) bound to the session. The session is --session, else AGENT_BROWSER_SESSION (T3 Code sets it per thread), else codex-, else - claude-. Prints the session, the link to its view - for the person and the agent-browser command to work in it. --json prints + claude-. Prints the session, the link to its tab in + the people's view (https://browser.…/t/), where the Operator works in + the same tab, and the agent-browser command to work in it. --json prints {kind: "browser-window", session, targetId, created, link, command}. browser link [--session ] [--folder ] [--json] - The link to the view of the session's window (or of every window without a - session), for the person: the Launchpad hands the browser over to the view. + The link for the person: with a session, the Launchpad's hand-over to that + session's tab; without one, the people's view of a new tab. Exit status: 0 done, 10 this Environment has no Environment browser, 2 - usage, 1 failure.`; + usage, 1 failure. +browser serve --port --origin + The people's view service, as lazurio-browser-view.service runs it; it runs + until it is stopped. Not for agents.`; const synopsis = "browser window|link [--session ] [--url ] [--folder ] [--json]"; @@ -66,7 +70,7 @@ const text = { "The window is open, but agent-browser did not bind the session to it.", failed: "lazurio browser failed", session: "Session", - view: "The person watches and takes over here", + view: "The Operator works with you in this tab", use: "Work in your window with", reused: "Your window is open already.", created: "Your window is open.", @@ -84,21 +88,24 @@ const text = { "Okno je otevřené, ale agent-browser k němu sezení nepřipojil.", failed: "lazurio browser selhal", session: "Sezení", - view: "Tady člověk sleduje a přebírá ovládání", + view: "Tady s tebou Operátor pracuje ve stejné záložce", use: "Ve svém okně pracuj příkazem", reused: "Tvoje okno už je otevřené.", created: "Tvoje okno je otevřené.", }, } as const; -/** The person's link: the Launchpad's hand-over to the view (it adds the - * view's access token itself), for one session or for all. */ +/** The person's link for `browser link`: the Launchpad's hand-over to a + * session's tab (it opens the window when the session has none), or the + * people's view of a new tab without a session. `browser window` prints the + * tab's own address instead (`browserViewUrl`). */ export function browserLink( launchpadOrigin: string, + browserOrigin: string, session: string | null, ): string { return session === null - ? `${launchpadOrigin}/.lazurio/browser` + ? browserViewUrl(browserOrigin, null) : `${launchpadOrigin}/.lazurio/browser?session=${session}`; } @@ -165,7 +172,8 @@ export async function runBrowserCommand( locale = state?.preferences.profile.locale === "cs" ? "cs" : "en"; const copy = text[locale]; const entry = state?.entry ?? null; - if (entry === null || browserEntryOf(entry) === undefined) + const browser = browserEntryOf(entry); + if (entry === null || browser === undefined) return Object.freeze({ code: exitUpdateAvailable, ...(json @@ -184,8 +192,8 @@ export async function runBrowserCommand( : threadSession(values.session, context.env); if (named === null && (action === "window" || values.session !== undefined)) return Object.freeze({ code: exitUsage, stderr: copy["no-session"] }); - const link = browserLink(entry.externalOrigin, named); if (action === "link") { + const link = browserLink(entry.externalOrigin, browser.origin, named); return Object.freeze({ code: exitOk, stdout: json @@ -200,6 +208,7 @@ export async function runBrowserCommand( context.windowSeams ?? cdpSeams(context.env, context.run ?? runProcess), ); const command = browserCommand(session); + const link = browserViewUrl(browser.origin, window.targetId); if (json) return Object.freeze({ code: exitOk, diff --git a/src/browser/people/bundle.ts b/src/browser/people/bundle.ts new file mode 100644 index 000000000..2ccaac1da --- /dev/null +++ b/src/browser/people/bundle.ts @@ -0,0 +1,26 @@ +// `/assets/view.js`, the script of the people's view (decision F39), as one +// ES module built from `client.ts` by the same Bun when this module is +// bundled: a macro (imported with `{ type: "macro" }`), so the compiled +// executable carries the script as a string, as it carries the shell's +// (`src/shell/bundle.ts`), and a source run builds it on load. Never called +// at run time of a compiled executable. +export function viewScript(): string { + const result = Bun.spawnSync( + [ + process.execPath, + "build", + `${import.meta.dir}/client.ts`, + "--target=browser", + "--format=esm", + "--minify", + ], + { + stdout: "pipe", + stderr: "pipe", + env: { PATH: process.env.PATH ?? "" }, + }, + ); + if (!result.success) + throw new Error(`The view script did not build: ${result.stderr}`); + return result.stdout.toString(); +} diff --git a/src/browser/people/cdp.ts b/src/browser/people/cdp.ts new file mode 100644 index 000000000..0f08c66cd --- /dev/null +++ b/src/browser/people/cdp.ts @@ -0,0 +1,173 @@ +/** One DevTools connection to the Environment browser for the people's view + * (decision F39): a WebSocket to the browser target on loopback, with pages + * reached through flattened sessions (`Target.attachToTarget` with + * `flatten`), so one socket carries every tab the view shows. */ + +export type CdpEvent = Readonly<{ + method: string; + params: Readonly>; + sessionId?: string; +}>; + +export type CdpConnection = Readonly<{ + send: ( + method: string, + params?: Readonly>, + sessionId?: string, + ) => Promise>; + /** Every event of the browser and of every attached session. */ + onEvent: (listener: (event: CdpEvent) => void) => void; + /** Once, when the socket closes for any reason. */ + onClose: (listener: () => void) => void; + close: () => void; +}>; + +export class CdpError extends Error { + constructor( + readonly method: string, + message: string, + ) { + super(`${method}: ${message}`); + } +} + +const callTimeoutMs = 15_000; + +/** The browser target's socket, from `/json/version` on loopback. Only a + * loopback `ws:` address on the same port is taken, so a changed answer can + * never send the view's DevTools traffic anywhere else. */ +export async function browserSocketUrl( + port: number, + fetcher: typeof fetch = fetch, +): Promise { + const response = await fetcher(`http://127.0.0.1:${port}/json/version`, { + redirect: "error", + signal: AbortSignal.timeout(3_000), + }); + if (!response.ok) throw new Error(`json/version answered ${response.status}`); + const body = (await response.json()) as { webSocketDebuggerUrl?: unknown }; + const url = body.webSocketDebuggerUrl; + if (typeof url !== "string") throw new Error("no webSocketDebuggerUrl"); + const parsed = new URL(url); + if ( + parsed.protocol !== "ws:" || + parsed.hostname !== "127.0.0.1" || + parsed.port !== String(port) || + !parsed.pathname.startsWith("/devtools/browser/") + ) + throw new Error("webSocketDebuggerUrl is not this loopback browser"); + return parsed.href; +} + +/** Opens the socket and resolves when it is open. Calls fail when the + * socket closes or after 15 s; nothing is retried here (the hub reconnects + * the whole connection). */ +export function connectCdp(url: string): Promise { + return new Promise((resolve, reject) => { + const socket = new WebSocket(url); + let nextId = 1; + const pending = new Map< + number, + { + method: string; + resolve: (value: Record) => void; + reject: (error: Error) => void; + timer: ReturnType; + } + >(); + const listeners: ((event: CdpEvent) => void)[] = []; + const closers: (() => void)[] = []; + let open = false; + let closed = false; + const finish = () => { + if (closed) return; + closed = true; + for (const [id, call] of pending) { + clearTimeout(call.timer); + call.reject(new CdpError(call.method, "connection closed")); + pending.delete(id); + } + for (const closer of closers.splice(0)) closer(); + }; + socket.addEventListener("open", () => { + open = true; + resolve( + Object.freeze({ + send: (method, params = {}, sessionId) => { + if (closed) + return Promise.reject(new CdpError(method, "connection closed")); + const id = nextId++; + return new Promise>( + (resolveCall, rejectCall) => { + const timer = setTimeout(() => { + pending.delete(id); + rejectCall(new CdpError(method, "timed out")); + }, callTimeoutMs); + pending.set(id, { + method, + resolve: resolveCall, + reject: rejectCall, + timer, + }); + socket.send( + JSON.stringify( + sessionId === undefined + ? { id, method, params } + : { id, method, params, sessionId }, + ), + ); + }, + ); + }, + onEvent: (listener) => { + listeners.push(listener); + }, + onClose: (listener) => { + if (closed) listener(); + else closers.push(listener); + }, + close: () => socket.close(), + }), + ); + }); + socket.addEventListener("message", (message) => { + let data: Record; + try { + data = JSON.parse(String(message.data)) as Record; + } catch { + return; + } + if (typeof data.id === "number") { + const call = pending.get(data.id); + if (call === undefined) return; + pending.delete(data.id); + clearTimeout(call.timer); + const error = data.error as { message?: unknown } | undefined; + if (error !== undefined) + call.reject(new CdpError(call.method, String(error.message))); + else call.resolve((data.result ?? {}) as Record); + return; + } + if (typeof data.method !== "string") return; + const event: CdpEvent = { + method: data.method, + params: (data.params ?? {}) as Record, + ...(typeof data.sessionId === "string" + ? { sessionId: data.sessionId } + : {}), + }; + for (const listener of listeners) { + try { + listener(event); + } catch {} + } + }); + socket.addEventListener("close", () => { + if (!open) reject(new Error("DevTools socket closed before it opened")); + finish(); + }); + socket.addEventListener("error", () => { + if (!open) reject(new Error("DevTools socket failed")); + }); + }); +} diff --git a/src/browser/people/client.ts b/src/browser/people/client.ts new file mode 100644 index 000000000..8403ab2c4 --- /dev/null +++ b/src/browser/people/client.ts @@ -0,0 +1,1331 @@ +import { + buttonBit, + clampInto, + clickCount, + containRect, + heldButton, + inputAction, + type KeyMessage, + keyAction, + mapPoint, + modifiersOf, + mouseButton, + type Point, + type Press, + textMessages, + viewSize, + wheelDelta, +} from "./input-map"; +import { + addressOf, + type ClientMessage, + decodeFrame, + type FrameHeader, + isTargetId, + type ServerMessage, + type ViewSize, +} from "./protocol"; + +// The script of the people's view of the Environment browser (decision +// F39), served as `/assets/view.js` to the page of `page.ts`: one tab of a +// person is a window into one remote tab. It opens a remote tab for `/`, +// draws the tab's frames, sends the person's pointer, touch and keys +// (input-map.ts), follows the tab in the bar, shows the tab's pop-ups over +// the page, offers its new tabs and answers its dialogs and file choosers. +// The gateway admits the person; the script carries no credential and talks +// only to its own origin (inside an app's frame it also tells that app where +// the tab is). + +const en = { + connecting: "Connecting…", + reconnecting: "The Environment browser is reconnecting…", + closed: "This tab no longer exists.", + openNewTab: "Open a new tab", + startFailed: "The new tab could not be opened.", + retry: "Try again", + newTab: "The page opened a new tab", + open: "Open", + dismiss: "Dismiss", + back: "Back", + forward: "Forward", + reload: "Reload", + stop: "Stop", + address: "Search or enter an address", + page: "Page in the Environment browser", + closeWindow: "Close the window", + ok: "OK", + cancel: "Cancel", + leaveTitle: "Leave this page?", + leaveText: "Changes you made may not be saved.", + stay: "Stay", + leave: "Leave", + file: "The page wants to upload a file", + files: "The page wants to upload files", + upload: "Upload", + uploading: "Uploading…", + uploadFailed: "The files could not be uploaded.", +}; +type Words = { readonly [K in keyof typeof en]: string }; +const cs: Words = { + connecting: "Připojuji…", + reconnecting: "Prohlížeč Environmentu se znovu připojuje…", + closed: "Tahle záložka už neexistuje.", + openNewTab: "Otevřít novou záložku", + startFailed: "Novou záložku se nepodařilo otevřít.", + retry: "Zkusit znovu", + newTab: "Stránka otevřela novou záložku", + open: "Otevřít", + dismiss: "Zavřít", + back: "Zpět", + forward: "Vpřed", + reload: "Obnovit", + stop: "Zastavit", + address: "Hledat nebo zadat adresu", + page: "Stránka v prohlížeči Environmentu", + closeWindow: "Zavřít okno", + ok: "OK", + cancel: "Zrušit", + leaveTitle: "Opustit stránku?", + leaveText: "Provedené změny nemusí být uložené.", + stay: "Zůstat", + leave: "Opustit", + file: "Stránka chce nahrát soubor", + files: "Stránka chce nahrát soubory", + upload: "Nahrát", + uploading: "Nahrávám…", + uploadFailed: "Soubory se nepodařilo nahrát.", +}; + +const czech = navigator.language.toLowerCase().startsWith("cs"); +const say: Words = czech ? cs : en; + +// The person's shortcuts use ⌘ on a Mac, an iPhone and an iPad (whose +// Safari says "Macintosh"). +const mac = /Mac|iPhone|iPad|iPod/.test(navigator.userAgent); + +// Inside an app's frame (web T3 Code's Browser panel, the Launchpad's right +// panel) the view tells the app where the tab is and which new tab to open. +// Only the app around it hears that; where the browser does not name it, any +// may, and the service lets only the Environment's own origins frame the view +// (F39 point 8). +const embedded = window.parent !== window; +const ancestor = location.ancestorOrigins?.item(0) ?? null; +const parentOrigin = ancestor !== null && ancestor !== "null" ? ancestor : "*"; +function tell(message: Readonly>): void { + if (!embedded) return; + try { + window.parent.postMessage(message, parentOrigin); + } catch {} +} + +function byId(id: string, kind: new () => T): T { + const node = document.getElementById(id); + if (!(node instanceof kind)) throw new Error(`The view page has no #${id}.`); + return node; +} + +function element( + tag: K, + className = "", + text = "", +): HTMLElementTagNameMap[K] { + const node = document.createElement(tag); + if (className !== "") node.className = className; + if (text !== "") node.textContent = text; + return node; +} + +function button( + text: string, + action: () => void, + className = "button", +): HTMLButtonElement { + const node = element("button", className, text); + node.type = "button"; + node.addEventListener("click", action); + return node; +} + +function label(node: HTMLElement, text: string): void { + node.setAttribute("aria-label", text); + node.title = text; +} + +const isHttps = (value: string) => { + try { + return new URL(value).protocol === "https:"; + } catch { + return false; + } +}; + +const hostOf = (url: string) => { + try { + return new URL(url).host || url; + } catch { + return url; + } +}; + +function serverMessage(data: string): ServerMessage | null { + try { + const value: unknown = JSON.parse(data); + return typeof value === "object" && + value !== null && + typeof (value as { t?: unknown }).t === "string" + ? (value as ServerMessage) + : null; + } catch { + return null; + } +} + +const back = byId("back", HTMLButtonElement); +const forward = byId("forward", HTMLButtonElement); +const reload = byId("reload", HTMLButtonElement); +const address = byId("address", HTMLInputElement); +const notice = byId("notice", HTMLDivElement); +const noticeText = byId("notice-text", HTMLSpanElement); +const noticeOpen = byId("notice-open", HTMLButtonElement); +const noticeClose = byId("notice-close", HTMLButtonElement); +const area = byId("area", HTMLElement); +const banner = byId("banner", HTMLDivElement); +const favicon = byId("icon", HTMLLinkElement); +const defaultIcon = favicon.href; + +document.documentElement.lang = czech ? "cs" : "en"; +label(back, say.back); +label(forward, say.forward); +label(reload, say.reload); +label(noticeClose, say.dismiss); +address.placeholder = say.address; +address.setAttribute("aria-label", say.address); +noticeOpen.textContent = say.open; +banner.textContent = say.reconnecting; + +type Info = Extract; +type Dialog = Extract; +type Files = Extract; +type Frame = Readonly<{ header: FrameHeader; jpeg: Uint8Array }>; +type Touch = { + id: number; + x: number; + y: number; + startX: number; + startY: number; + tap: boolean; +}; +type Wheel = Readonly<{ + x: number; + y: number; + deltaX: number; + deltaY: number; + buttons: number; + modifiers: number; +}>; + +type ViewerOptions = Readonly<{ + /** The person's own tab, whose window takes the page area's size (F39 + * point 3); false for a pop-up over it, which keeps the size its opener + * gave it. */ + resize: boolean; + info: (info: Info) => void; + /** The remote tab no longer exists. */ + closed: () => void; + /** The remote viewport changed size (a pop-up's panel follows it). */ + viewport?: (width: number, height: number) => void; +}>; + +const firstRetryMs = 500; +const lastRetryMs = 10_000; +const sizeDelayMs = 150; +// A touch that moves further is a gesture, not a tap. +const tapDistance = 10; +const modifierKeys = new Set([ + "Shift", + "Control", + "Alt", + "AltGraph", + "Meta", + "CapsLock", +]); +const sizeKey = (size: ViewSize) => `${size.width}x${size.height}@${size.dpr}`; + +/** One remote tab in one box of the page: its socket, its frames and the + * person's input to it. The page's own tab and each pop-up over it are one + * viewer each. */ +class Viewer { + readonly box = element("div", "page"); + private readonly screen = element("canvas", "screen"); + // The person's keys go to this field while they work in the page: hidden, + // focusable, so it also receives `paste`, `copy`, `cut`, input methods and + // a phone's keyboard. + private readonly keys = element("textarea", "keys"); + private readonly status = element("div", "status", say.connecting); + private readonly context: CanvasRenderingContext2D | null; + private readonly observer: ResizeObserver; + private socket: WebSocket | null = null; + private retry: ReturnType | null = null; + private retryMs = firstRetryMs; + private ended = false; + private bitmap: ImageBitmap | null = null; + private header: FrameHeader | null = null; + private waiting: Frame | null = null; + private decoding = false; + private sizeTimer: ReturnType | null = null; + private sentSize = ""; + private selection = ""; + /** Keys whose key down the remote page had, by the person's `code`. */ + private readonly down = new Map(); + private composing = false; + /** The person's buttons as last reported, and whether the remote page + * has a press of this view that is not released yet. */ + private buttons = 0; + private pressed = false; + private press: Press | null = null; + /** The newest move (mouse or touch) and the wheel's sum, sent once per + * animation frame. */ + private moving: ClientMessage | null = null; + private wheel: Wheel | null = null; + private frameRequest = 0; + private readonly touches = new Map(); + private dialog: HTMLDialogElement | null = null; + private chooser: Readonly<{ + dialog: HTMLDialogElement; + token: string; + abort: AbortController; + }> | null = null; + + constructor( + readonly id: string, + parent: HTMLElement, + private readonly options: ViewerOptions, + ) { + this.context = this.screen.getContext("2d"); + const keys = this.keys; + keys.value = " "; + keys.setAttribute("aria-label", say.page); + keys.setAttribute("autocapitalize", "off"); + keys.setAttribute("autocomplete", "off"); + keys.setAttribute("autocorrect", "off"); + keys.setAttribute("spellcheck", "false"); + keys.setAttribute("wrap", "off"); + this.status.setAttribute("role", "status"); + this.box.append(this.screen, keys, this.status); + parent.append(this.box); + this.listen(); + this.observer = new ResizeObserver(() => this.resized()); + this.observer.observe(this.box); + } + + start(): void { + this.connect(); + } + + /** For good: no reconnect, the socket and the modals closed. The last + * frame stays drawn. */ + stop(): void { + this.ended = true; + if (this.retry !== null) clearTimeout(this.retry); + if (this.sizeTimer !== null) clearTimeout(this.sizeTimer); + if (this.frameRequest !== 0) cancelAnimationFrame(this.frameRequest); + this.retry = null; + this.sizeTimer = null; + this.frameRequest = 0; + this.observer.disconnect(); + const socket = this.socket; + this.socket = null; + socket?.close(); + this.closeDialog(); + this.closeChooser(null); + this.showStatus(); + } + + /** Sends when connected; input while the socket is down is dropped (the + * remote page moves on, F39 failure modes). */ + send(message: ClientMessage): boolean { + const socket = this.socket; + if (socket === null || socket.readyState !== WebSocket.OPEN) return false; + socket.send(JSON.stringify(message)); + return true; + } + + /** The person's keys go into this tab. */ + focus(): void { + if (this.ended) return; + this.keys.focus({ preventScroll: true }); + this.resetKeys(); + } + + // ---- The socket ---------------------------------------------------------- + + private connect(): void { + this.retry = null; + if (this.ended) return; + const scheme = location.protocol === "https:" ? "wss" : "ws"; + const socket = new WebSocket( + `${scheme}://${location.host}/t/${this.id}/live`, + ); + socket.binaryType = "arraybuffer"; + this.socket = socket; + socket.addEventListener("open", () => { + if (this.socket !== socket) return; + const size = this.size(); + this.sentSize = sizeKey(size); + this.send({ t: "hello", ...size, resize: this.options.resize }); + this.showStatus(); + }); + socket.addEventListener("message", (event: MessageEvent) => { + if (this.socket !== socket) return; + this.retryMs = firstRetryMs; + if (typeof event.data === "string") this.message(event.data); + else if (event.data instanceof ArrayBuffer) { + const frame = decodeFrame(event.data); + if (frame !== null) this.frame(frame); + } + }); + socket.addEventListener("close", () => { + if (this.socket !== socket) return; + this.socket = null; + // What the remote page had down went with the socket. + this.down.clear(); + this.touches.clear(); + this.pressed = false; + this.showStatus(); + if (this.ended) return; + this.retry = setTimeout(() => this.connect(), this.retryMs); + this.retryMs = Math.min(this.retryMs * 2, lastRetryMs); + }); + } + + private message(data: string): void { + const message = serverMessage(data); + if (message === null) return; + switch (message.t) { + case "info": + this.options.info(message); + return; + case "selection": + this.selection = message.text; + return; + case "popup": + openPopup(message.id, message.url); + return; + case "new-tab": + offerNewTab(message.id, message.url); + return; + case "dialog": + this.showDialog(message); + return; + case "dialog-closed": + this.closeDialog(); + return; + case "files": + this.showChooser(message); + return; + case "files-closed": + this.closeChooser(message.token); + return; + case "browser": + banner.hidden = message.state !== "reconnecting"; + return; + case "closed": + this.stop(); + this.options.closed(); + return; + } + } + + private showStatus(): void { + this.status.hidden = + this.ended || + (this.socket?.readyState === WebSocket.OPEN && this.bitmap !== null); + } + + // ---- Frames and size ----------------------------------------------------- + + private frame(frame: Frame): void { + // While one frame decodes only the newest waits: it is the current one. + this.waiting = frame; + if (!this.decoding) void this.decode(); + } + + private async decode(): Promise { + this.decoding = true; + for (let frame = this.waiting; frame !== null; frame = this.waiting) { + this.waiting = null; + // The frame's bytes are a view of the socket's own ArrayBuffer. + const jpeg = frame.jpeg as Uint8Array; + const bitmap = await createImageBitmap( + new Blob([jpeg], { type: "image/jpeg" }), + ).catch(() => null); + if (bitmap === null) continue; + const before = this.header; + this.bitmap?.close(); + this.bitmap = bitmap; + this.header = frame.header; + this.draw(); + this.showStatus(); + if ( + before?.deviceWidth !== frame.header.deviceWidth || + before?.deviceHeight !== frame.header.deviceHeight + ) + this.options.viewport?.( + frame.header.deviceWidth, + frame.header.deviceHeight, + ); + } + this.decoding = false; + } + + private draw(): void { + const context = this.context; + if (context === null) return; + context.clearRect(0, 0, this.screen.width, this.screen.height); + if (this.bitmap === null) return; + const drawn = containRect(this.screen, this.bitmap); + context.imageSmoothingQuality = "high"; + context.drawImage( + this.bitmap, + drawn.left, + drawn.top, + drawn.width, + drawn.height, + ); + } + + private size(): ViewSize { + return viewSize( + this.box.clientWidth, + this.box.clientHeight, + window.devicePixelRatio || 1, + ); + } + + /** The canvas follows the box at once; the remote window follows it a + * moment later (`size`), once the box has stopped changing. */ + private resized(): void { + const ratio = window.devicePixelRatio || 1; + const width = Math.max(1, Math.round(this.box.clientWidth * ratio)); + const height = Math.max(1, Math.round(this.box.clientHeight * ratio)); + if (this.screen.width !== width || this.screen.height !== height) { + this.screen.width = width; + this.screen.height = height; + this.draw(); + } + if (this.sizeTimer !== null) clearTimeout(this.sizeTimer); + this.sizeTimer = setTimeout(() => { + this.sizeTimer = null; + const size = this.size(); + if (sizeKey(size) === this.sentSize) return; + if (this.send({ t: "size", ...size })) this.sentSize = sizeKey(size); + }, sizeDelayMs); + } + + // ---- Pointer and touch --------------------------------------------------- + + private listen(): void { + const screen = this.screen; + screen.addEventListener("pointerdown", (event) => { + event.preventDefault(); + if (event.pointerType === "touch") this.touchStart(event); + else this.mouseDown(event); + }); + screen.addEventListener("pointermove", (event) => { + if (event.pointerType === "touch") this.touchMove(event); + else this.mouseMove(event); + }); + screen.addEventListener("pointerup", (event) => { + event.preventDefault(); + if (event.pointerType === "touch") this.touchEnd(event, false); + else this.button(event, "mouseReleased", true); + }); + screen.addEventListener("pointercancel", (event) => { + if (event.pointerType === "touch") this.touchEnd(event, true); + }); + // The compatibility mouse events would take the focus off the keys, and + // a back or forward button would take the person's own tab back. + for (const type of [ + "mousedown", + "mouseup", + "auxclick", + "contextmenu", + "dragstart", + ] as const) + screen.addEventListener(type, (event) => event.preventDefault()); + screen.addEventListener("wheel", (event) => this.wheeled(event), { + passive: false, + }); + this.listenToKeys(); + } + + /** The event's place on the remote page; with `clamp`, a place outside + * the frame counts as its nearest edge (a drag that left the frame). */ + private at(event: MouseEvent, clamp: boolean): Point | null { + if (this.bitmap === null || this.header === null) return null; + const box = this.screen.getBoundingClientRect(); + const inner = containRect(box, this.bitmap); + const drawn = { + left: box.left + inner.left, + top: box.top + inner.top, + width: inner.width, + height: inner.height, + }; + const client = { x: event.clientX, y: event.clientY }; + return mapPoint( + clamp ? clampInto(client, drawn) : client, + drawn, + this.header, + ); + } + + private mouseDown(event: PointerEvent): void { + this.focus(); + this.placeKeys(event); + this.capture(event); + this.button(event, "mousePressed", false); + } + + /** Keeps a drag's moves on this view when it leaves the frame. A pointer + * that is no longer active cannot be captured (NotFoundError); the press + * still goes to the remote page. */ + private capture(event: PointerEvent): void { + try { + this.screen.setPointerCapture(event.pointerId); + } catch {} + } + + private mouseMove(event: PointerEvent): void { + // A second button pressed or released while another is held comes as a + // move (chorded buttons). + if (event.buttons !== this.buttons && event.button >= 0) { + const pressed = (event.buttons & buttonBit(event.button)) !== 0; + this.button(event, pressed ? "mousePressed" : "mouseReleased", true); + return; + } + const at = this.at(event, this.pressed); + if (at === null) return; + this.moving = { + t: "mouse", + type: "mouseMoved", + x: at.x, + y: at.y, + button: heldButton(event.buttons), + buttons: event.buttons, + clickCount: 0, + modifiers: modifiersOf(event, mac), + }; + this.schedule(); + } + + /** A press or a release, after the move before it. A release only of a + * press the remote page had; a press outside the frame is none. */ + private button( + event: PointerEvent, + type: "mousePressed" | "mouseReleased", + clamp: boolean, + ): void { + this.buttons = event.buttons; + if (type === "mouseReleased" && !this.pressed) return; + const at = this.at(event, clamp); + if (at === null) return; + this.flush(); + const button = mouseButton(event.button); + if (type === "mousePressed") { + const press = { + time: event.timeStamp, + x: event.clientX, + y: event.clientY, + button, + }; + this.press = { ...press, count: clickCount(this.press, press) }; + } + const sent = this.send({ + t: "mouse", + type, + x: at.x, + y: at.y, + button, + buttons: event.buttons, + clickCount: this.press?.button === button ? this.press.count : 1, + modifiers: modifiersOf(event, mac), + }); + this.pressed = sent && (type === "mousePressed" || event.buttons !== 0); + } + + private wheeled(event: WheelEvent): void { + event.preventDefault(); + const at = this.at(event, false); + if (at === null || this.header === null) return; + const delta = wheelDelta(event, this.header.deviceHeight); + const before = this.wheel; + this.wheel = { + x: at.x, + y: at.y, + deltaX: (before?.deltaX ?? 0) + delta.deltaX, + deltaY: (before?.deltaY ?? 0) + delta.deltaY, + buttons: event.buttons, + modifiers: modifiersOf(event, mac), + }; + this.schedule(); + } + + private schedule(): void { + if (this.frameRequest !== 0) return; + this.frameRequest = requestAnimationFrame(() => { + this.frameRequest = 0; + this.flush(); + }); + } + + /** Sends the waiting move and wheel now: before a press, a release or a + * touch that follows them. */ + private flush(): void { + if (this.frameRequest !== 0) cancelAnimationFrame(this.frameRequest); + this.frameRequest = 0; + const moving = this.moving; + this.moving = null; + if (moving !== null) this.send(moving); + const wheel = this.wheel; + this.wheel = null; + if (wheel === null) return; + const delta = wheelDelta({ ...wheel, deltaMode: 0 }, 0); + this.send({ + t: "mouse", + type: "mouseWheel", + x: wheel.x, + y: wheel.y, + button: "none", + buttons: wheel.buttons, + clickCount: 0, + modifiers: wheel.modifiers, + deltaX: delta.deltaX, + deltaY: delta.deltaY, + }); + } + + private touchStart(event: PointerEvent): void { + if (this.touches.size >= 10) return; + const at = this.at(event, false); + if (at === null) return; + this.capture(event); + const ids = new Set([...this.touches.values()].map((touch) => touch.id)); + let id = 0; + while (ids.has(id)) id += 1; + // A second finger makes a gesture of the first, not a tap. + for (const touch of this.touches.values()) touch.tap = false; + this.touches.set(event.pointerId, { + id, + x: at.x, + y: at.y, + startX: event.clientX, + startY: event.clientY, + tap: this.touches.size === 0, + }); + this.flush(); + this.send(this.touchMessage("touchStart", event)); + } + + private touchMove(event: PointerEvent): void { + const touch = this.touches.get(event.pointerId); + if (touch === undefined) return; + const at = this.at(event, true); + if (at === null) return; + touch.x = at.x; + touch.y = at.y; + if ( + Math.hypot(event.clientX - touch.startX, event.clientY - touch.startY) > + tapDistance + ) + touch.tap = false; + this.moving = this.touchMessage("touchMove", event); + this.schedule(); + } + + private touchEnd(event: PointerEvent, cancelled: boolean): void { + const touch = this.touches.get(event.pointerId); + if (touch === undefined) return; + this.touches.delete(event.pointerId); + this.flush(); + // While other fingers stay, the remote lifts the one that is gone from + // the points it no longer gets. + this.send( + this.touchMessage( + this.touches.size > 0 + ? "touchMove" + : cancelled + ? "touchCancel" + : "touchEnd", + event, + ), + ); + // A tap opens the phone's keyboard (F39 point 6); only a focus within + // the tap itself does. + if (!cancelled && touch.tap) { + this.placeKeys(event); + this.focus(); + } + } + + private touchMessage( + type: "touchStart" | "touchMove" | "touchEnd" | "touchCancel", + event: PointerEvent, + ): ClientMessage { + return { + t: "touch", + type, + // An end or a cancel carries no point (CDP's rule). + points: + type === "touchEnd" || type === "touchCancel" + ? [] + : [...this.touches.values()].map(({ id, x, y }) => ({ id, x, y })), + modifiers: modifiersOf(event, mac), + }; + } + + // ---- Keys and the clipboard ---------------------------------------------- + + private listenToKeys(): void { + const keys = this.keys; + keys.addEventListener("keydown", (event) => this.key(event)); + keys.addEventListener("keyup", (event) => this.key(event)); + keys.addEventListener("focus", () => this.resetKeys()); + // Key ups go elsewhere now: what the remote page has down goes up. + keys.addEventListener("blur", () => this.releaseKeys(false, 0)); + keys.addEventListener("paste", (event) => { + event.preventDefault(); + this.sendAll( + textMessages(event.clipboardData?.getData("text/plain") ?? ""), + ); + }); + keys.addEventListener("copy", (event) => this.copy(event)); + keys.addEventListener("cut", (event) => this.copy(event)); + // A phone's keyboard: its keys come as `keyCode` 229, their edits here. + keys.addEventListener("beforeinput", (event) => { + const messages = inputAction( + event.inputType, + event.data ?? event.dataTransfer?.getData("text/plain") ?? null, + ); + if (messages === null) return; + if (event.cancelable) event.preventDefault(); + this.sendAll(messages); + }); + keys.addEventListener("compositionstart", () => { + this.composing = true; + }); + keys.addEventListener("compositionend", (event) => { + this.composing = false; + this.sendAll(textMessages(event.data)); + setTimeout(() => { + if (!this.composing) this.resetKeys(); + }, 0); + }); + keys.addEventListener("input", (event) => { + if (this.composing || (event instanceof InputEvent && event.isComposing)) + return; + this.resetKeys(); + }); + } + + private sendAll(messages: readonly ClientMessage[]): void { + for (const message of messages) this.send(message); + } + + private key(event: KeyboardEvent): void { + const action = keyAction( + { + type: event.type === "keyup" ? "keyup" : "keydown", + key: event.key, + code: event.code, + keyCode: event.keyCode, + ctrlKey: event.ctrlKey, + metaKey: event.metaKey, + altKey: event.altKey, + shiftKey: event.shiftKey, + altGraph: event.getModifierState("AltGraph"), + location: event.location, + repeat: event.repeat, + isComposing: event.isComposing, + }, + mac, + ); + if (action.preventDefault) event.preventDefault(); + const message = action.send; + if (message === null) return; + if (message.type === "keyDown") { + if (this.send(message)) this.down.set(event.code, message); + return; + } + // A key up only for a key the remote page saw go down. + if (!this.down.delete(event.code)) return; + this.send(message); + // While ⌘ is held a Mac reports no key up for the other keys: they go up + // with it. + if (mac && event.key === "Meta") this.releaseKeys(true, message.modifiers); + } + + /** Key ups for what the remote page has down: every key, or with + * `keysOnly` all but the modifiers. */ + private releaseKeys(keysOnly: boolean, modifiers: number): void { + for (const [code, down] of this.down) { + if (keysOnly && modifierKeys.has(down.key)) continue; + this.down.delete(code); + this.send({ + t: "key", + type: "keyUp", + key: down.key, + code: down.code, + keyCode: down.keyCode, + modifiers, + location: down.location, + repeat: false, + }); + } + } + + /** Copy and cut put the remote selection on the person's clipboard (F39 + * point 6). With nothing selected the clipboard is not given the field's + * space. */ + private copy(event: ClipboardEvent): void { + event.preventDefault(); + if (this.selection !== "") + event.clipboardData?.setData("text/plain", this.selection); + } + + /** One selected space: browsers fire `copy` and `cut` only with a + * selection, and a phone's Backspace needs something to delete. */ + private resetKeys(): void { + if (this.keys.value !== " ") this.keys.value = " "; + this.keys.setSelectionRange(0, 1); + } + + /** The keys' field under the pointer, where an input method shows its + * candidates and a phone scrolls to. */ + private placeKeys(event: MouseEvent): void { + const box = this.box.getBoundingClientRect(); + this.keys.style.left = `${Math.round(event.clientX - box.left)}px`; + this.keys.style.top = `${Math.round(event.clientY - box.top)}px`; + } + + // ---- Dialogs and file choosers ------------------------------------------- + + private showModal(dialog: HTMLDialogElement): void { + document.body.append(dialog); + dialog.showModal(); + } + + /** A JavaScript dialog of the remote page, answered here (F39 point 7). */ + private showDialog(message: Dialog): void { + this.closeDialog(); + const dialog = element("dialog", "modal"); + const leave = message.kind === "beforeunload"; + let field: HTMLInputElement | null = null; + const answer = (accept: boolean) => { + this.send( + accept && field !== null + ? { t: "dialog", accept, text: field.value } + : { t: "dialog", accept }, + ); + this.closeDialog(); + }; + if (leave) + dialog.append( + element("h2", "", say.leaveTitle), + element("p", "message", say.leaveText), + ); + // The page's words as text, never as markup. + else dialog.append(element("p", "message", message.message)); + if (message.kind === "prompt") { + const input = element("input", "field"); + input.type = "text"; + input.value = message.defaultPrompt; + input.addEventListener("keydown", (event) => { + if (event.key !== "Enter" || event.isComposing) return; + event.preventDefault(); + answer(true); + }); + dialog.append(input); + field = input; + } + const ok = button( + leave ? say.leave : say.ok, + () => answer(true), + "button primary", + ); + const actions = element("div", "actions"); + if (message.kind !== "alert") + actions.append( + button(leave ? say.stay : say.cancel, () => answer(false)), + ); + actions.append(ok); + dialog.append(actions); + // Escape cancels; an alert has only OK. + dialog.addEventListener("cancel", (event) => { + event.preventDefault(); + answer(message.kind === "alert"); + }); + this.dialog = dialog; + this.showModal(dialog); + if (field === null) ok.focus(); + else { + field.focus(); + field.select(); + } + } + + private closeDialog(): void { + const dialog = this.dialog; + if (dialog === null) return; + this.dialog = null; + dialog.close(); + dialog.remove(); + this.focus(); + } + + /** The remote page asked for files: the person picks them here and they + * are uploaded to the Environment for the page's input (F39 point 7). */ + private showChooser(message: Files): void { + this.closeChooser(null); + const { token } = message; + const abort = new AbortController(); + const dialog = element("dialog", "modal"); + const input = element("input"); + input.type = "file"; + input.multiple = message.multiple; + const error = element("p", "error", say.uploadFailed); + error.hidden = true; + const cancel = () => { + this.send({ t: "files-cancel", token }); + this.closeChooser(token); + }; + const upload = button(say.upload, () => void submit(), "button primary"); + upload.disabled = true; + input.addEventListener("change", () => { + upload.disabled = (input.files?.length ?? 0) === 0; + }); + const submit = async () => { + const form = new FormData(); + for (const file of input.files ?? []) form.append("file", file); + upload.disabled = true; + input.disabled = true; + upload.textContent = say.uploading; + error.hidden = true; + const done = await fetch( + `/t/${this.id}/files?token=${encodeURIComponent(token)}`, + { + method: "POST", + body: form, + credentials: "same-origin", + signal: abort.signal, + }, + ).then( + (response) => response.ok, + () => false, + ); + if (abort.signal.aborted) return; + if (done) { + this.closeChooser(token); + return; + } + upload.disabled = false; + input.disabled = false; + upload.textContent = say.upload; + error.hidden = false; + }; + const actions = element("div", "actions"); + actions.append(button(say.cancel, cancel), upload); + dialog.append( + element("h2", "", message.multiple ? say.files : say.file), + input, + error, + actions, + ); + dialog.addEventListener("cancel", (event) => { + event.preventDefault(); + cancel(); + }); + this.chooser = { dialog, token, abort }; + this.showModal(dialog); + input.focus(); + } + + /** Closes the file chooser: any, or only the one of `token` (it was + * answered, here or by another view). */ + private closeChooser(token: string | null): void { + const chooser = this.chooser; + if (chooser === null || (token !== null && chooser.token !== token)) return; + this.chooser = null; + chooser.abort.abort(); + chooser.dialog.close(); + chooser.dialog.remove(); + this.focus(); + } +} + +// ---- The page's own tab -------------------------------------------------- + +let main: Viewer | null = null; +/** The remote tab's address as last reported, and whether it loads. */ +let current = ""; +let loading = false; +/** The address bar: just focused (its text selected), typed in since, just + * submitted. Until the person types, the tab's address replaces its text, + * as in a browser's own bar. */ +let justFocused = false; +let edited = false; +let submitted = false; +let offered: string | null = null; + +const shown = (url: string) => (url === "about:blank" ? "" : url); + +function showInfo(info: Info): void { + current = info.url; + if (!edited) { + address.value = shown(info.url); + if (document.activeElement === address) address.select(); + } + back.disabled = !info.canGoBack; + forward.disabled = !info.canGoForward; + loading = info.loading; + reload.dataset.state = loading ? "stop" : "reload"; + label(reload, loading ? say.stop : say.reload); + document.title = info.title || info.url; + const icon = + info.favicon !== null && isHttps(info.favicon) ? info.favicon : defaultIcon; + if (favicon.href !== icon) favicon.href = icon; + tell({ + type: "lazurio-browser:info", + url: info.url, + title: info.title, + view: location.href, + }); +} + +function navigate(action: "back" | "forward" | "reload" | "stop"): void { + main?.send({ t: "nav", action }); + main?.focus(); +} + +back.addEventListener("click", () => navigate("back")); +forward.addEventListener("click", () => navigate("forward")); +reload.addEventListener("click", () => navigate(loading ? "stop" : "reload")); + +// Focus selects the whole address; the click that focused it would place +// the caret instead, so its mouseup keeps the selection. +address.addEventListener("focus", () => { + address.select(); + justFocused = true; +}); +address.addEventListener("mouseup", (event) => { + if (justFocused) event.preventDefault(); + justFocused = false; +}); +address.addEventListener("input", () => { + edited = true; +}); +address.addEventListener("keydown", (event) => { + justFocused = false; + if (event.isComposing) return; + if (event.key === "Enter") { + event.preventDefault(); + const value = address.value.trim(); + if (value === "" || main === null) return; + const url = addressOf(value); + submitted = true; + address.value = url; + main.send({ t: "goto", url }); + main.focus(); + } else if (event.key === "Escape") { + event.preventDefault(); + address.value = shown(current); + main?.focus(); + } +}); +address.addEventListener("blur", () => { + justFocused = false; + if (!submitted) address.value = shown(current); + edited = false; + submitted = false; +}); + +/** A page's new tab without access to its opener (F39 point 4): offered with + * one click, and to the app around the view. */ +function offerNewTab(id: string, url: string): void { + if (!isTargetId(id)) return; + offered = id; + noticeText.textContent = `${say.newTab}: ${url}`; + noticeText.title = url; + notice.hidden = false; + tell({ + type: "lazurio-browser:new-tab", + view: new URL(`/t/${id}`, location.href).href, + url, + }); +} + +function dismissNotice(): void { + notice.hidden = true; + offered = null; + main?.focus(); +} + +noticeOpen.addEventListener("click", () => { + if (offered !== null) window.open(`/t/${offered}`, "_blank", "noopener"); + dismissNotice(); +}); +noticeClose.addEventListener("click", dismissNotice); + +// ---- Pop-ups ------------------------------------------------------------- + +// Pop-ups lie over the page, each in a panel of its own size, the newest on +// top (F39 point 4); around them the page stays usable, as around a pop-up +// window. +const popupLayer = element("div", "popups"); +const popups = new Map< + string, + Readonly<{ panel: HTMLElement; viewer: Viewer; fit: () => void }> +>(); +new ResizeObserver(() => { + for (const popup of popups.values()) popup.fit(); +}).observe(popupLayer); + +function openPopup(id: string, url: string): void { + if (!isTargetId(id) || popups.has(id)) return; + const panel = element("section", "popup"); + panel.setAttribute("role", "dialog"); + panel.setAttribute("aria-label", hostOf(url)); + const title = element("span", "popup-title", hostOf(url)); + const close = button("", () => viewer.send({ t: "close" }), "tool"); + label(close, say.closeWindow); + const cross = noticeClose.querySelector("svg"); + if (cross !== null) close.append(cross.cloneNode(true)); + const head = element("div", "popup-head"); + head.append(title, close); + const body = element("div", "popup-body"); + panel.append(head, body); + popupLayer.append(panel); + // The pop-up's own size, as its opener chose it; smaller only where the + // page area is. + let wanted = { width: 480, height: 600 }; + const fit = () => { + const room = popupLayer.getBoundingClientRect(); + const scale = Math.min( + 1, + (room.width - 32) / wanted.width, + (room.height - 32 - head.offsetHeight) / wanted.height, + ); + body.style.width = `${Math.max(1, Math.floor(wanted.width * scale))}px`; + body.style.height = `${Math.max(1, Math.floor(wanted.height * scale))}px`; + }; + const viewer = new Viewer(id, body, { + resize: false, + info: (info) => { + title.textContent = hostOf(info.url); + panel.setAttribute("aria-label", title.textContent); + }, + closed: () => closePopup(id), + viewport: (width, height) => { + wanted = { width, height }; + fit(); + }, + }); + popups.set(id, { panel, viewer, fit }); + fit(); + viewer.start(); + viewer.focus(); +} + +function closePopup(id: string): void { + const popup = popups.get(id); + if (popup === undefined) return; + popups.delete(id); + const focused = popup.panel.contains(document.activeElement); + popup.viewer.stop(); + popup.panel.remove(); + if (focused) ([...popups.values()].at(-1)?.viewer ?? main)?.focus(); +} + +// ---- Start --------------------------------------------------------------- + +/** A message over the page area with one action. */ +function showEnd(text: string, action: string, act: () => void): HTMLElement { + const panel = element("div", "end"); + const go = button(action, act, "button primary"); + panel.append(element("p", "", text), go); + area.append(panel); + go.focus(); + return panel; +} + +function tabClosed(): void { + for (const control of [back, forward, reload, address]) + control.disabled = true; + showEnd(say.closed, say.openNewTab, () => location.assign("/")); +} + +/** A new remote tab at the page area's size (F39 point 1); null when the + * service opened none. */ +async function createTab(): Promise { + try { + const response = await fetch("/api/tabs", { + method: "POST", + credentials: "same-origin", + headers: { "content-type": "application/json" }, + body: JSON.stringify( + viewSize( + area.clientWidth, + area.clientHeight, + window.devicePixelRatio || 1, + ), + ), + }); + if (response.status !== 201) return null; + const body: unknown = await response.json(); + const id = + typeof body === "object" && body !== null + ? (body as { id?: unknown }).id + : undefined; + return isTargetId(id) ? id : null; + } catch { + return null; + } +} + +function open(id: string, fresh: boolean): void { + main = new Viewer(id, area, { + resize: true, + info: showInfo, + closed: tabClosed, + }); + area.append(popupLayer); + main.start(); + // A new tab waits for an address, as a browser's new tab does. + if (fresh) address.focus(); + else main.focus(); +} + +/** `/t/` views that tab; `/` opens a new one and takes its address. */ +async function start(): Promise { + const viewed = /^\/t\/([^/]+)$/.exec(location.pathname)?.[1]; + if (viewed !== undefined || location.pathname !== "/") { + if (viewed !== undefined && isTargetId(viewed)) open(viewed, false); + else tabClosed(); + return; + } + const waiting = element("div", "status", say.connecting); + waiting.setAttribute("role", "status"); + area.append(waiting); + const id = await createTab(); + waiting.remove(); + if (id === null) { + const panel = showEnd(say.startFailed, say.retry, () => { + panel.remove(); + void start(); + }); + return; + } + history.replaceState(null, "", `/t/${id}`); + open(id, true); +} + +void start(); diff --git a/src/browser/people/extension.ts b/src/browser/people/extension.ts new file mode 100644 index 000000000..3b199e37b --- /dev/null +++ b/src/browser/people/extension.ts @@ -0,0 +1,132 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { writeDurableFile } from "../../update/durable-file"; + +/** The Lazurio extension of the Environment browser (decision F39 point 7), + * loaded with `--load-extension` from a directory next to the profile, never + * inside it. Two jobs, no permissions: + * - a tab a page opens in a window that already has one moves into a window + * of its own, because Chrome paints only visible windows and the people's + * view shows one tab per person's tab (point 2); + * - until the Environment's Bitwarden holds its passkeys (0191 point 17), a + * WebAuthn request is declined at once, so Chrome's own passkey window, + * which the view cannot show and which blocks the page, never opens. */ + +export const browserExtensionDirectory = (home: string) => + join(home, ".local", "share", "lazurio-browser", "extension"); + +const manifest = `${JSON.stringify( + { + manifest_version: 3, + name: "Lazurio Environment browser", + version: "1.0.0", + description: + "Keeps every tab of the Environment browser in a window of its own and declines passkey requests until the Environment's Bitwarden holds its passkeys.", + background: { service_worker: "background.js" }, + content_scripts: [ + { + matches: [""], + js: ["webauthn.js"], + run_at: "document_start", + all_frames: true, + match_about_blank: true, + world: "MAIN", + }, + ], + }, + null, + 2, +)}\n`; + +const background = `// Lazurio (decision F39): a tab opened in a window that already has a tab +// moves into a window of its own. Chrome paints only visible windows, and the +// people's view shows exactly one tab per person's tab. The tab keeps its +// opener, so a page and the tab it opened still talk to each other. +chrome.tabs.onCreated.addListener(async (tab) => { + try { + if (tab.id === undefined || tab.windowId === undefined) return; + const window = await chrome.windows.get(tab.windowId); + if (window.type !== "normal") return; + const tabs = await chrome.tabs.query({ windowId: tab.windowId }); + if (tabs.length < 2) return; + await chrome.windows.create({ tabId: tab.id, focused: false }); + } catch {} +}); +`; + +const webauthn = `// Lazurio (decision F39, root decision 0191 point 17): until the +// Environment's Bitwarden holds its passkeys, a passkey request is declined at +// once. Chrome's own passkey window opens outside the page, where the people's +// view cannot show it, and the page takes no input until it is closed. A +// declined request lets the site offer another way to sign in. +(() => { + const Container = globalThis.CredentialsContainer; + if (!Container) return; + const decline = () => + Promise.reject( + new DOMException( + "Passkeys are not available in the Environment browser yet.", + "NotAllowedError", + ), + ); + const get = Container.prototype.get; + const create = Container.prototype.create; + Container.prototype.get = function (options) { + return options && options.publicKey ? decline() : get.call(this, options); + }; + Container.prototype.create = function (options) { + return options && options.publicKey ? decline() : create.call(this, options); + }; + const Credential = globalThis.PublicKeyCredential; + if (!Credential) return; + Credential.isUserVerifyingPlatformAuthenticatorAvailable = () => + Promise.resolve(false); + if (Credential.isConditionalMediationAvailable) + Credential.isConditionalMediationAvailable = () => Promise.resolve(false); + if (Credential.getClientCapabilities) + Credential.getClientCapabilities = () => + Promise.resolve({ + conditionalCreate: false, + conditionalGet: false, + hybridTransport: false, + passkeyPlatformAuthenticator: false, + userVerifyingPlatformAuthenticator: false, + }); +})(); +`; + +export const browserExtensionFiles: Readonly> = + Object.freeze({ + "manifest.json": manifest, + "background.js": background, + "webauthn.js": webauthn, + }); + +/** One digest of every file, carried in the browser unit's text, so a new + * extension changes the unit and restarts the browser (Chrome reads an + * unpacked extension when it starts). */ +export const browserExtensionDigest = createHash("sha256") + .update( + Object.entries(browserExtensionFiles) + .map(([name, text]) => `${name}\0${text}`) + .join("\0"), + ) + .digest("hex") + .slice(0, 16); + +/** Writes the files that differ. True when anything was written. */ +export async function writeBrowserExtension(home: string): Promise { + const directory = browserExtensionDirectory(home); + await mkdir(directory, { recursive: true, mode: 0o700 }); + let changed = false; + for (const [name, text] of Object.entries(browserExtensionFiles)) { + const existing = await readFile(join(directory, name), "utf8").catch( + () => undefined, + ); + if (existing === text) continue; + await writeDurableFile(directory, name, Buffer.from(text)); + changed = true; + } + return changed; +} diff --git a/src/browser/people/hub.ts b/src/browser/people/hub.ts new file mode 100644 index 000000000..1013e3113 --- /dev/null +++ b/src/browser/people/hub.ts @@ -0,0 +1,1232 @@ +import { randomBytes } from "node:crypto"; +import { + mkdir, + readdir, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { basename, join } from "node:path"; +import type { CdpConnection, CdpEvent } from "./cdp"; +import { + type ClientMessage, + encodeFrame, + type FrameHeader, + type ServerMessage, + type ViewSize, + viewSizeLimits, +} from "./protocol"; + +/** The view service's hold on the Environment browser (decision F39): every + * page target with one DevTools session each, the people who view which tab, + * and what the view does for them (frames, input, pop-ups, dialogs, file + * choosers, closing an unused tab). One hub per service; it reconnects + * whenever the browser goes away. */ + +/** One person's socket on one tab. */ +export type Viewer = { + send: (message: ServerMessage) => void; + sendFrame: (frame: Uint8Array) => void; + /** Bytes the socket has not sent yet. */ + buffered: () => number; + close: () => void; + size: ViewSize | null; + resize: boolean; +}; + +export type HubSeams = Readonly<{ + /** Opens a DevTools connection to the browser target on loopback. */ + connect: () => Promise; + /** The target ids agent-browser sessions are bound to right now. */ + boundTargets: () => Promise>; + /** Where uploaded files wait until the page reads them. */ + uploadDirectory: string; + /** The virtual screen; no window grows past it. */ + screen: Readonly<{ width: number; height: number }>; + log: (line: string) => void; + /** Overridable for tests. */ + graceMs?: number; + unviewedGraceMs?: number; + reconnectMs?: number; +}>; + +type Info = { + url: string; + title: string; + favicon: string | null; + canGoBack: boolean; + canGoForward: boolean; + loading: boolean; +}; + +type Tab = { + targetId: string; + sessionId: string | null; + openerId: string | null; + canAccessOpener: boolean; + mainFrameId: string | null; + info: Info; + viewers: Set; + screencast: { width: number; height: number } | null; + pendingAck: number | null; + lastFrame: Uint8Array | null; + /** A screencast frame came since the tab was last watched: the first + * screenshot, if it is slower, is older and is not shown. */ + live: boolean; + selection: string; + dialog: Extract | null; + lastActor: Viewer | null; + resizeTimer: ReturnType | null; + infoTimer: ReturnType | null; + graceTimer: ReturnType | null; + watched: boolean; + attaching: boolean; + /** The view's isolated world in the main frame, until it navigates. */ + world: number | null; + /** A page opened by another, announced once its address is known. */ + pendingOpen: { + openerId: string; + timer: ReturnType; + } | null; + selectionTimer: ReturnType | null; + /** Reads the title and icon while the tab is watched. */ + metaTimer: ReturnType | null; +}; + +type FileChooser = { + targetId: string; + backendNodeId: number; + multiple: boolean; +}; + +const viewWorld = "lazurio-view"; +// The page side of copy and cut (F39 point 6): the remote selection, read in +// an isolated world the page's own scripts cannot reach, whenever it may have +// changed (a released mouse button, a released key) and when the person +// presses Ctrl or ⌘, just before a copy. Read on demand rather than reported +// through a binding: a binding needs the Runtime domain enabled, which pages +// can detect as automation, and a person's sign-in must not look like one. A +// password field's selection is never sent. +const selectionScript = `(() => { + const el = document.activeElement; + let text = ""; + if (el && (el.tagName === "TEXTAREA" || el.tagName === "INPUT") && typeof el.selectionStart === "number") { + text = el.type === "password" ? "" : String(el.value).slice(el.selectionStart, el.selectionEnd); + } else { + text = String(document.getSelection() ?? ""); + } + return text.length > 1000000 ? text.slice(0, 1000000) : text; +})()`; +// The page's title and icon. Chrome reports a target's title only as its +// address until the next navigation (no Target.targetInfoChanged for a +// `` or a later change), so the view reads both from the page. +const metaScript = `(() => { + const link = document.querySelector('link[rel~="icon"]'); + let icon = ""; + try { icon = link ? link.href : new URL("/favicon.ico", location.href).href; } catch {} + return JSON.stringify({ title: String(document.title ?? ""), icon }); +})()`; +const metaEveryMs = 2_000; + +const ackThreshold = 1 << 20; +const jpegQuality = 70; + +/** The size of a JPEG from its first start-of-frame marker; null when the + * bytes are not a JPEG the parser understands. */ +export function jpegSize( + bytes: Uint8Array, +): { width: number; height: number } | null { + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null; + let i = 2; + while (i + 9 < bytes.length) { + if (bytes[i] !== 0xff) return null; + const marker = bytes[i + 1] as number; + const length = ((bytes[i + 2] as number) << 8) | (bytes[i + 3] as number); + if ( + marker >= 0xc0 && + marker <= 0xcf && + marker !== 0xc4 && + marker !== 0xc8 && + marker !== 0xcc + ) + return { + height: ((bytes[i + 5] as number) << 8) | (bytes[i + 6] as number), + width: ((bytes[i + 7] as number) << 8) | (bytes[i + 8] as number), + }; + i += 2 + length; + } + return null; +} + +const clamp = (value: number, min: number, max: number) => + Math.min(max, Math.max(min, value)); + +export function clampSize(size: ViewSize): ViewSize { + return { + width: Math.round( + clamp(size.width, viewSizeLimits.minWidth, viewSizeLimits.maxWidth), + ), + height: Math.round( + clamp(size.height, viewSizeLimits.minHeight, viewSizeLimits.maxHeight), + ), + dpr: clamp(size.dpr, 1, viewSizeLimits.maxDpr), + }; +} + +/** Only these addresses are navigated to from the address bar; the page + * itself may go anywhere it likes. */ +export function navigableAddress(value: string): string | null { + if (value === "about:blank") return value; + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:" + ? url.href + : null; + } catch { + return null; + } +} + +export class BrowserHub { + private cdp: CdpConnection | null = null; + private stopped = false; + private reconnectTimer: ReturnType<typeof setTimeout> | null = null; + private readonly tabs = new Map<string, Tab>(); + private readonly bySession = new Map<string, string>(); + /** Tabs the view opened, or a page opened from one: the only tabs the + * view ever closes (F39 point 9). Lost on restart, on purpose. */ + private readonly ownTabs = new Set<string>(); + private readonly fileChoosers = new Map<string, FileChooser>(); + /** Page targets discovery reported on the current connection. */ + private seen = new Set<string>(); + private readonly graceMs: number; + private readonly unviewedGraceMs: number; + private readonly reconnectMs: number; + + constructor(private readonly seams: HubSeams) { + this.graceMs = seams.graceMs ?? 30_000; + this.unviewedGraceMs = seams.unviewedGraceMs ?? 600_000; + this.reconnectMs = seams.reconnectMs ?? 2_000; + } + + get connected(): boolean { + return this.cdp !== null; + } + + start(): void { + this.stopped = false; + void this.connect(); + } + + stop(): void { + this.stopped = true; + if (this.reconnectTimer !== null) clearTimeout(this.reconnectTimer); + this.cdp?.close(); + this.cdp = null; + for (const tab of this.tabs.values()) this.clearTimers(tab); + } + + /** A new window of the default context, at the person's size when given. + * Null when the browser cannot be reached. */ + async createTab(size: ViewSize | null): Promise<string | null> { + const cdp = this.cdp; + if (cdp === null) return null; + const fitted = size === null ? null : this.windowSize(clampSize(size)); + try { + const result = await cdp.send("Target.createTarget", { + url: "about:blank", + newWindow: true, + ...(fitted === null + ? {} + : { width: fitted.width, height: fitted.height }), + }); + const targetId = String(result.targetId); + this.ownTabs.add(targetId); + const deadline = Date.now() + 2_000; + while (!this.tabs.has(targetId) && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 20)); + const tab = this.tabs.get(targetId); + if (tab === undefined) return null; + if (tab.viewers.size === 0) this.scheduleGrace(tab, this.unviewedGraceMs); + return targetId; + } catch (error) { + this.seams.log(`create tab failed: ${String(error)}`); + return null; + } + } + + hasTab(targetId: string): boolean { + return this.tabs.has(targetId); + } + + /** Adds a person's socket to a tab. False when there is no such page. */ + attach(targetId: string, viewer: Viewer): boolean { + const tab = this.tabs.get(targetId); + if (tab === undefined) return false; + tab.viewers.add(viewer); + if (tab.graceTimer !== null) { + clearTimeout(tab.graceTimer); + tab.graceTimer = null; + } + viewer.send({ + t: "browser", + state: + this.cdp !== null && tab.sessionId !== null + ? "connected" + : "reconnecting", + }); + this.sendInfo(tab, viewer); + if (tab.selection !== "") + viewer.send({ t: "selection", text: tab.selection }); + if (tab.dialog !== null) viewer.send(tab.dialog); + if (tab.lastFrame !== null) viewer.sendFrame(tab.lastFrame); + void this.watch(tab); + return true; + } + + detach(targetId: string, viewer: Viewer): void { + const tab = this.tabs.get(targetId); + if (tab === undefined) return; + tab.viewers.delete(viewer); + if (tab.lastActor === viewer) tab.lastActor = null; + if (tab.viewers.size > 0) { + void this.updateScreencast(tab); + return; + } + void this.unwatch(tab); + if (this.ownTabs.has(targetId)) this.scheduleGrace(tab, this.graceMs); + } + + /** Called when a viewer's socket drained, to release a held frame. */ + drained(targetId: string): void { + const tab = this.tabs.get(targetId); + if (tab !== undefined) this.maybeAck(tab); + } + + async handle( + targetId: string, + viewer: Viewer, + message: ClientMessage, + ): Promise<void> { + const tab = this.tabs.get(targetId); + if (tab === undefined) return; + if (message.t === "hello" || message.t === "size") { + viewer.size = clampSize(message); + if (message.t === "hello") viewer.resize = message.resize; + if (viewer.resize) tab.lastActor = viewer; + this.scheduleResize(tab); + await this.updateScreencast(tab); + return; + } + // A person may act before the tab's session is attached (a new tab, a + // reconnect): wait for it briefly rather than drop the input. + const deadline = Date.now() + 3_000; + while ( + (this.cdp === null || tab.sessionId === null) && + Date.now() < deadline && + this.tabs.get(targetId) === tab + ) + await new Promise((resolve) => setTimeout(resolve, 25)); + const cdp = this.cdp; + if (cdp === null || tab.sessionId === null) return; + const session = tab.sessionId; + const call = (method: string, params: Record<string, unknown> = {}) => + cdp.send(method, params, session).catch((error: unknown) => { + this.seams.log(`${method} failed: ${String(error)}`); + return null; + }); + switch (message.t) { + case "mouse": { + this.acted(tab, viewer); + if (message.type === "mouseReleased") this.selectionSoon(tab); + await call("Input.dispatchMouseEvent", { + type: message.type, + x: message.x, + y: message.y, + button: message.button, + buttons: message.buttons, + clickCount: message.clickCount, + modifiers: message.modifiers, + ...(message.type === "mouseWheel" + ? { deltaX: message.deltaX ?? 0, deltaY: message.deltaY ?? 0 } + : {}), + }); + return; + } + case "key": { + this.acted(tab, viewer); + if ( + message.type === "keyUp" || + message.key === "Control" || + message.key === "Meta" + ) + this.selectionSoon(tab); + const typing = message.type === "keyDown" && message.text !== undefined; + await call("Input.dispatchKeyEvent", { + type: typing + ? "keyDown" + : message.type === "keyDown" + ? "rawKeyDown" + : "keyUp", + key: message.key, + code: message.code, + windowsVirtualKeyCode: message.keyCode, + nativeVirtualKeyCode: message.keyCode, + modifiers: message.modifiers, + location: message.location, + autoRepeat: message.repeat, + isKeypad: message.location === 3, + ...(typing + ? { text: message.text, unmodifiedText: message.text } + : {}), + }); + return; + } + case "text": + this.acted(tab, viewer); + await call("Input.insertText", { text: message.text }); + return; + case "touch": + this.acted(tab, viewer); + await call("Input.dispatchTouchEvent", { + type: message.type, + touchPoints: message.points.map((point) => ({ + x: point.x, + y: point.y, + id: point.id, + })), + modifiers: message.modifiers, + }); + return; + case "nav": { + if (message.action === "reload") await call("Page.reload"); + else if (message.action === "stop") await call("Page.stopLoading"); + else { + const history = await call("Page.getNavigationHistory"); + const index = Number(history?.currentIndex); + const entries = (history?.entries ?? []) as { id: number }[]; + const entry = + entries[message.action === "back" ? index - 1 : index + 1]; + if (entry !== undefined) + await call("Page.navigateToHistoryEntry", { entryId: entry.id }); + } + return; + } + case "goto": { + const url = navigableAddress(message.url); + if (url !== null) await call("Page.navigate", { url }); + return; + } + case "dialog": + await call("Page.handleJavaScriptDialog", { + accept: message.accept, + ...(message.text === undefined ? {} : { promptText: message.text }), + }); + return; + case "files-cancel": { + const chooser = this.fileChoosers.get(message.token); + if (chooser === undefined || chooser.targetId !== targetId) return; + this.fileChoosers.delete(message.token); + this.broadcast(tab, { t: "files-closed", token: message.token }); + return; + } + case "close": + await cdp + .send("Target.closeTarget", { targetId }) + .catch(() => undefined); + return; + } + } + + /** Puts uploaded files on the input that asked for them. False when the + * token is unknown or the browser refused. */ + async upload( + targetId: string, + token: string, + files: readonly File[], + ): Promise<boolean> { + const chooser = this.fileChoosers.get(token); + const tab = this.tabs.get(targetId); + const cdp = this.cdp; + if ( + chooser === undefined || + chooser.targetId !== targetId || + tab === undefined || + tab.sessionId === null || + cdp === null || + files.length === 0 || + (!chooser.multiple && files.length > 1) + ) + return false; + this.fileChoosers.delete(token); + await this.sweepUploads(); + const directory = join(this.seams.uploadDirectory, token); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const paths: string[] = []; + for (const [index, file] of files.entries()) { + const name = basename(file.name).replace(/[^\w.@+-]/g, "_") || "file"; + const folder = join(directory, String(index)); + await mkdir(folder, { mode: 0o700 }); + const path = join(folder, name.slice(0, 200)); + await writeFile(path, new Uint8Array(await file.arrayBuffer()), { + mode: 0o600, + }); + paths.push(path); + } + try { + await cdp.send( + "DOM.setFileInputFiles", + { files: paths, backendNodeId: chooser.backendNodeId }, + tab.sessionId, + ); + } catch (error) { + this.seams.log(`setFileInputFiles failed: ${String(error)}`); + return false; + } + this.broadcast(tab, { t: "files-closed", token }); + return true; + } + + // ---- The connection ------------------------------------------------------ + + private async connect(): Promise<void> { + if (this.stopped) return; + let cdp: CdpConnection; + try { + cdp = await this.seams.connect(); + } catch (error) { + this.seams.log(`browser unreachable: ${String(error)}`); + this.retry(); + return; + } + if (this.stopped) { + cdp.close(); + return; + } + this.cdp = cdp; + cdp.onEvent((event) => this.event(event)); + cdp.onClose(() => { + if (this.cdp !== cdp) return; + this.cdp = null; + this.lost(); + this.retry(); + }); + this.seen = new Set(); + try { + // Chrome reports every existing target before it answers. + await cdp.send("Target.setDiscoverTargets", { discover: true }); + } catch (error) { + this.seams.log(`discover failed: ${String(error)}`); + cdp.close(); + return; + } + this.seams.log("browser connected"); + // A tab kept across a reconnect whose page did not come back is gone. + for (const id of [...this.tabs.keys()]) + if (!this.seen.has(id)) this.gone(id); + for (const id of [...this.ownTabs]) + if (!this.seen.has(id)) this.ownTabs.delete(id); + } + + private retry(): void { + if (this.stopped || this.reconnectTimer !== null) return; + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + void this.connect(); + }, this.reconnectMs); + } + + /** The browser went away: every view waits, sessions are void. */ + private lost(): void { + this.seams.log("browser connection lost"); + this.bySession.clear(); + for (const tab of this.tabs.values()) { + tab.sessionId = null; + tab.screencast = null; + tab.pendingAck = null; + tab.watched = false; + tab.attaching = false; + tab.world = null; + if (tab.metaTimer !== null) clearInterval(tab.metaTimer); + tab.metaTimer = null; + for (const viewer of tab.viewers) + viewer.send({ t: "browser", state: "reconnecting" }); + } + // A target id does not survive a browser restart; ids that come back + // are attached again by discovery. + for (const [id, tab] of [...this.tabs]) + if (tab.viewers.size === 0) { + this.clearTimers(tab); + this.tabs.delete(id); + } + } + + private event(event: CdpEvent): void { + const p = event.params; + switch (event.method) { + case "Target.targetCreated": + case "Target.targetInfoChanged": { + const info = p.targetInfo as Record<string, unknown> | undefined; + if (info?.type !== "page") return; + this.page(info, event.method === "Target.targetCreated"); + return; + } + case "Target.targetDestroyed": + this.gone(String(p.targetId)); + return; + case "Target.detachedFromTarget": { + const targetId = this.bySession.get(String(p.sessionId)); + this.bySession.delete(String(p.sessionId)); + const tab = + targetId === undefined ? undefined : this.tabs.get(targetId); + if (tab !== undefined) { + tab.sessionId = null; + tab.screencast = null; + } + return; + } + } + if (event.sessionId === undefined) return; + const targetId = this.bySession.get(event.sessionId); + const tab = targetId === undefined ? undefined : this.tabs.get(targetId); + if (tab === undefined) return; + this.pageEvent(tab, event); + } + + private page(info: Record<string, unknown>, created: boolean): void { + const targetId = String(info.targetId); + let tab = this.tabs.get(targetId); + const url = String(info.url ?? ""); + const title = String(info.title ?? ""); + if (tab === undefined) { + const openerId = typeof info.openerId === "string" ? info.openerId : null; + tab = { + targetId, + sessionId: null, + openerId, + canAccessOpener: info.canAccessOpener === true, + mainFrameId: null, + info: { + url, + title, + favicon: null, + canGoBack: false, + canGoForward: false, + loading: false, + }, + viewers: new Set(), + screencast: null, + pendingAck: null, + lastFrame: null, + live: false, + selection: "", + dialog: null, + lastActor: null, + resizeTimer: null, + infoTimer: null, + graceTimer: null, + watched: false, + attaching: false, + world: null, + pendingOpen: null, + selectionTimer: null, + metaTimer: null, + }; + this.tabs.set(targetId, tab); + if (created && openerId !== null) this.opened(tab, openerId); + if (this.ownTabs.has(targetId) && tab.viewers.size === 0) + this.scheduleGrace(tab, this.unviewedGraceMs); + } else { + // Chrome's target title is only the address (see metaScript): it is + // taken with a new address, and the page's own title follows. + if (url !== tab.info.url) tab.info.title = title; + tab.info.url = url; + this.queueInfo(tab); + if (tab.pendingOpen !== null && url !== "" && url !== "about:blank") + this.announce(tab, tab.pendingOpen.openerId); + } + this.seen.add(targetId); + if (tab.sessionId === null) void this.attachSession(tab); + } + + /** A page opened by another (F39 point 4): a pop-up over its opener's + * view, or a new tab to offer. Chrome reports it before it has an + * address, so it is announced at its first address, or after 1.5 s. */ + private opened(tab: Tab, openerId: string): void { + if (!this.tabs.has(openerId)) return; + if (this.ownTabs.has(openerId)) { + this.ownTabs.add(tab.targetId); + this.scheduleGrace(tab, this.unviewedGraceMs); + } + if (tab.info.url !== "" && tab.info.url !== "about:blank") { + this.announce(tab, openerId); + return; + } + tab.pendingOpen = { + openerId, + timer: setTimeout(() => this.announce(tab, openerId), 1_500), + }; + } + + private announce(tab: Tab, openerId: string): void { + if (tab.pendingOpen !== null) clearTimeout(tab.pendingOpen.timer); + tab.pendingOpen = null; + const opener = this.tabs.get(openerId); + if (opener === undefined || !this.tabs.has(tab.targetId)) return; + this.broadcast( + opener, + tab.canAccessOpener + ? { t: "popup", id: tab.targetId, url: tab.info.url } + : { t: "new-tab", id: tab.targetId, url: tab.info.url }, + ); + } + + private async attachSession(tab: Tab): Promise<void> { + const cdp = this.cdp; + if (cdp === null || tab.sessionId !== null || tab.attaching) return; + tab.attaching = true; + let sessionId: string; + try { + const result = await cdp.send("Target.attachToTarget", { + targetId: tab.targetId, + flatten: true, + }); + sessionId = String(result.sessionId); + } catch { + tab.attaching = false; + return; + } + tab.attaching = false; + if (this.cdp !== cdp || !this.tabs.has(tab.targetId)) return; + tab.sessionId = sessionId; + this.bySession.set(sessionId, tab.targetId); + const call = (method: string, params: Record<string, unknown> = {}) => + cdp.send(method, params, sessionId); + try { + // Every page, watched or not: dialogs and file choosers are never + // left to Chrome's own windows (F39 point 7). + await call("Page.enable"); + await call("Page.setInterceptFileChooserDialog", { enabled: true }); + const tree = await call("Page.getFrameTree"); + const frame = (tree.frameTree as { frame?: { id?: unknown } })?.frame; + tab.mainFrameId = typeof frame?.id === "string" ? frame.id : null; + await this.history(tab); + } catch (error) { + this.seams.log(`attach ${tab.targetId} incomplete: ${String(error)}`); + } + if (tab.viewers.size > 0) { + this.broadcast(tab, { t: "browser", state: "connected" }); + await this.watch(tab); + } + } + + // ---- A watched tab ------------------------------------------------------- + + /** First viewer: focus without stealing the window, the selection + * reporter, frames. */ + private async watch(tab: Tab): Promise<void> { + const cdp = this.cdp; + const session = tab.sessionId; + if (cdp === null || session === null || tab.watched) return; + tab.watched = true; + tab.live = false; + const call = (method: string, params: Record<string, unknown> = {}) => + cdp.send(method, params, session).catch((error: unknown) => { + this.seams.log(`${method} failed: ${String(error)}`); + return null; + }); + await call("Emulation.setFocusEmulationEnabled", { enabled: true }); + await this.meta(tab); + if (tab.metaTimer === null && tab.watched) + tab.metaTimer = setInterval(() => void this.meta(tab), metaEveryMs); + await this.firstFrame(tab); + await this.updateScreencast(tab); + } + + private async unwatch(tab: Tab): Promise<void> { + const cdp = this.cdp; + const session = tab.sessionId; + tab.watched = false; + if (tab.metaTimer !== null) clearInterval(tab.metaTimer); + tab.metaTimer = null; + // A frame of a tab nobody views is not kept: the next viewer gets a + // fresh screenshot (`firstFrame`). + tab.lastFrame = null; + if (tab.resizeTimer !== null) clearTimeout(tab.resizeTimer); + tab.resizeTimer = null; + if (cdp === null || session === null || tab.screencast === null) return; + tab.screencast = null; + tab.pendingAck = null; + await cdp.send("Page.stopScreencast", {}, session).catch(() => undefined); + } + + /** Runs a script in the view's isolated world of the main frame: one + * world per document, made again after a navigation or when Chrome no + * longer knows it. */ + private async inWorld(tab: Tab, expression: string): Promise<unknown> { + const cdp = this.cdp; + const session = tab.sessionId; + if (cdp === null || session === null || tab.mainFrameId === null) + return undefined; + for (let attempt = 0; attempt < 2; attempt++) { + try { + if (tab.world === null) { + const world = await cdp.send( + "Page.createIsolatedWorld", + { frameId: tab.mainFrameId, worldName: viewWorld }, + session, + ); + tab.world = Number(world.executionContextId); + } + const result = await cdp.send( + "Runtime.evaluate", + { expression, contextId: tab.world, returnByValue: true }, + session, + ); + return (result.result as { value?: unknown } | undefined)?.value; + } catch { + tab.world = null; + } + } + return undefined; + } + + /** Reads the remote selection soon, once for a burst of input. */ + private selectionSoon(tab: Tab): void { + if (tab.selectionTimer !== null) return; + tab.selectionTimer = setTimeout(() => { + tab.selectionTimer = null; + void this.inWorld(tab, selectionScript).then((value) => { + if (typeof value !== "string" || value === tab.selection) return; + tab.selection = value; + this.broadcast(tab, { t: "selection", text: value }); + }); + }, 40); + } + + private async meta(tab: Tab): Promise<void> { + const value = await this.inWorld(tab, metaScript); + if (typeof value !== "string") return; + let read: { title?: unknown; icon?: unknown }; + try { + read = JSON.parse(value) as { title?: unknown; icon?: unknown }; + } catch { + return; + } + const title = + typeof read.title === "string" ? read.title.slice(0, 500) : ""; + const icon = + typeof read.icon === "string" && read.icon.startsWith("https://") + ? read.icon + : null; + if (title !== "" && title !== tab.info.title) { + tab.info.title = title; + this.queueInfo(tab); + } + if (icon !== tab.info.favicon) { + tab.info.favicon = icon; + this.queueInfo(tab); + } + } + + private async history(tab: Tab): Promise<void> { + const cdp = this.cdp; + if (cdp === null || tab.sessionId === null) return; + try { + const history = await cdp.send( + "Page.getNavigationHistory", + {}, + tab.sessionId, + ); + const index = Number(history.currentIndex); + const count = Array.isArray(history.entries) ? history.entries.length : 0; + tab.info.canGoBack = index > 0; + tab.info.canGoForward = index < count - 1; + this.queueInfo(tab); + } catch {} + } + + /** A screenshot right away, so a new viewer does not wait for the page's + * next paint. */ + private async firstFrame(tab: Tab): Promise<void> { + const cdp = this.cdp; + if (cdp === null || tab.sessionId === null) return; + try { + const [shot, metrics] = await Promise.all([ + cdp.send( + "Page.captureScreenshot", + { format: "jpeg", quality: jpegQuality }, + tab.sessionId, + ), + cdp.send("Page.getLayoutMetrics", {}, tab.sessionId), + ]); + const viewport = metrics.cssVisualViewport as + | { clientWidth?: number; clientHeight?: number } + | undefined; + const jpeg = Buffer.from(String(shot.data), "base64"); + const size = jpegSize(jpeg); + if (size === null || viewport === undefined || tab.live) return; + this.deliver( + tab, + { + width: size.width, + height: size.height, + deviceWidth: Number(viewport.clientWidth), + deviceHeight: Number(viewport.clientHeight), + offsetTop: 0, + pageScaleFactor: 1, + }, + jpeg, + ); + } catch {} + } + + /** (Re)starts the screencast at the largest size any viewer shows. */ + private async updateScreencast(tab: Tab): Promise<void> { + const cdp = this.cdp; + const session = tab.sessionId; + if (cdp === null || session === null || !tab.watched) return; + // Only viewers that said their size (`hello` comes right after the + // socket opens): no screencast at a guessed size to restart a moment + // later. + let width = 0; + let height = 0; + for (const viewer of tab.viewers) { + const size = viewer.size; + if (size === null) continue; + width = Math.max(width, Math.round(size.width * size.dpr)); + height = Math.max(height, Math.round(size.height * size.dpr)); + } + if (width === 0) return; + width = Math.min(width, viewSizeLimits.maxWidth); + height = Math.min(height, viewSizeLimits.maxHeight); + if ( + tab.screencast !== null && + tab.screencast.width === width && + tab.screencast.height === height + ) + return; + if (tab.screencast !== null) + await cdp.send("Page.stopScreencast", {}, session).catch(() => undefined); + tab.screencast = { width, height }; + tab.pendingAck = null; + await cdp + .send( + "Page.startScreencast", + { + format: "jpeg", + quality: jpegQuality, + maxWidth: width, + maxHeight: height, + everyNthFrame: 1, + }, + session, + ) + .catch((error: unknown) => { + tab.screencast = null; + this.seams.log(`startScreencast failed: ${String(error)}`); + }); + } + + private deliver(tab: Tab, header: FrameHeader, jpeg: Uint8Array): void { + const frame = encodeFrame(header, jpeg); + tab.lastFrame = frame; + for (const viewer of tab.viewers) { + // A viewer that is far behind skips frames; the next one is current. + if (viewer.buffered() < 4 * ackThreshold) viewer.sendFrame(frame); + } + } + + /** Chrome sends the next frame only after an ack: ack when every viewer + * has drained, so a slow viewer slows Chrome down, not the service's + * memory (F39 failure modes). */ + private maybeAck(tab: Tab): void { + const cdp = this.cdp; + if (cdp === null || tab.sessionId === null || tab.pendingAck === null) + return; + for (const viewer of tab.viewers) + if (viewer.buffered() >= ackThreshold) return; + const ack = tab.pendingAck; + tab.pendingAck = null; + void cdp + .send("Page.screencastFrameAck", { sessionId: ack }, tab.sessionId) + .catch(() => undefined); + } + + private pageEvent(tab: Tab, event: CdpEvent): void { + const p = event.params; + switch (event.method) { + case "Page.screencastFrame": { + const metadata = (p.metadata ?? {}) as Record<string, number>; + const jpeg = Buffer.from(String(p.data), "base64"); + const size = jpegSize(jpeg); + tab.pendingAck = Number(p.sessionId); + tab.live = true; + if (size !== null && tab.watched) + this.deliver( + tab, + { + width: size.width, + height: size.height, + deviceWidth: Number(metadata.deviceWidth), + deviceHeight: Number(metadata.deviceHeight), + offsetTop: Number(metadata.offsetTop ?? 0), + pageScaleFactor: Number(metadata.pageScaleFactor ?? 1), + }, + jpeg, + ); + this.maybeAck(tab); + return; + } + case "Page.frameNavigated": { + const frame = p.frame as { id?: string; parentId?: string } | undefined; + if (frame?.parentId !== undefined) return; + if (typeof frame?.id === "string") tab.mainFrameId = frame.id; + tab.world = null; + tab.selection = ""; + void this.history(tab); + return; + } + case "Page.navigatedWithinDocument": + void this.history(tab); + if (tab.watched) void this.meta(tab); + return; + case "Page.frameStartedLoading": + case "Page.frameStoppedLoading": { + if (p.frameId !== tab.mainFrameId) return; + tab.info.loading = event.method === "Page.frameStartedLoading"; + this.queueInfo(tab); + return; + } + case "Page.loadEventFired": + case "Page.domContentEventFired": + if (tab.watched) void this.meta(tab); + return; + case "Page.javascriptDialogOpening": { + const type = String(p.type); + tab.dialog = { + t: "dialog", + kind: + type === "confirm" || type === "prompt" || type === "beforeunload" + ? type + : "alert", + message: String(p.message ?? ""), + defaultPrompt: String(p.defaultPrompt ?? ""), + }; + this.broadcast(tab, tab.dialog); + return; + } + case "Page.javascriptDialogClosed": + tab.dialog = null; + this.broadcast(tab, { t: "dialog-closed" }); + return; + case "Page.fileChooserOpened": { + if (tab.viewers.size === 0) { + // Nobody to pick a file: dropped, never Chrome's own window. + this.seams.log(`file chooser on ${tab.targetId} with no viewer`); + return; + } + const token = randomBytes(12).toString("hex"); + this.fileChoosers.set(token, { + targetId: tab.targetId, + backendNodeId: Number(p.backendNodeId), + multiple: p.mode === "selectMultiple", + }); + this.broadcast(tab, { + t: "files", + multiple: p.mode === "selectMultiple", + token, + }); + return; + } + } + } + + // ---- Window size --------------------------------------------------------- + + private acted(tab: Tab, viewer: Viewer): void { + if (!viewer.resize || tab.lastActor === viewer) return; + tab.lastActor = viewer; + this.scheduleResize(tab); + } + + private scheduleResize(tab: Tab): void { + if (tab.resizeTimer !== null) clearTimeout(tab.resizeTimer); + tab.resizeTimer = setTimeout(() => { + tab.resizeTimer = null; + void this.resize(tab); + }, 150); + } + + /** The window around a page area of this size, within the screen. The + * browser's own bars are added by `resize`, which measures them. */ + private windowSize(size: ViewSize): { width: number; height: number } { + return { + width: Math.min(size.width, this.seams.screen.width), + height: Math.min(size.height, this.seams.screen.height), + }; + } + + /** F39 point 3: the window takes the last acting person's page area. */ + private async resize(tab: Tab): Promise<void> { + const cdp = this.cdp; + const actor = tab.lastActor; + if ( + cdp === null || + tab.sessionId === null || + actor === null || + actor.size === null || + !actor.resize + ) + return; + try { + const [win, metrics] = await Promise.all([ + cdp.send("Browser.getWindowForTarget", { targetId: tab.targetId }), + cdp.send("Page.getLayoutMetrics", {}, tab.sessionId), + ]); + const bounds = win.bounds as { width: number; height: number }; + const viewport = metrics.cssLayoutViewport as { + clientWidth: number; + clientHeight: number; + }; + const barsWidth = Math.max(0, bounds.width - viewport.clientWidth); + const barsHeight = Math.max(0, bounds.height - viewport.clientHeight); + const width = Math.min( + actor.size.width + barsWidth, + this.seams.screen.width, + ); + const height = Math.min( + actor.size.height + barsHeight, + this.seams.screen.height, + ); + if ( + Math.abs(width - bounds.width) <= 2 && + Math.abs(height - bounds.height) <= 2 + ) + return; + await cdp.send("Browser.setWindowBounds", { + windowId: win.windowId, + bounds: { width, height }, + }); + } catch (error) { + this.seams.log(`resize ${tab.targetId} failed: ${String(error)}`); + } + } + + // ---- Lifecycle ----------------------------------------------------------- + + private scheduleGrace(tab: Tab, ms: number): void { + if (tab.graceTimer !== null) clearTimeout(tab.graceTimer); + tab.graceTimer = setTimeout(() => { + tab.graceTimer = null; + void this.closeIfUnused(tab.targetId); + }, ms); + } + + /** F39 point 9: an own tab nobody views and no agent-browser session is + * bound to. */ + private async closeIfUnused(targetId: string): Promise<void> { + const tab = this.tabs.get(targetId); + const cdp = this.cdp; + if ( + tab === undefined || + cdp === null || + tab.viewers.size > 0 || + !this.ownTabs.has(targetId) + ) + return; + const bound = await this.seams + .boundTargets() + .catch(() => new Set<string>([targetId])); + if (bound.has(targetId) || tab.viewers.size > 0) return; + this.seams.log(`closing unused tab ${targetId}`); + await cdp.send("Target.closeTarget", { targetId }).catch(() => undefined); + } + + private gone(targetId: string): void { + const tab = this.tabs.get(targetId); + if (tab === undefined) return; + this.clearTimers(tab); + this.tabs.delete(targetId); + this.ownTabs.delete(targetId); + if (tab.sessionId !== null) this.bySession.delete(tab.sessionId); + for (const [token, chooser] of this.fileChoosers) + if (chooser.targetId === targetId) this.fileChoosers.delete(token); + for (const viewer of tab.viewers) { + viewer.send({ t: "closed" }); + viewer.close(); + } + } + + private clearTimers(tab: Tab): void { + for (const timer of [ + tab.graceTimer, + tab.resizeTimer, + tab.infoTimer, + tab.selectionTimer, + tab.pendingOpen?.timer ?? null, + ]) + if (timer !== null) clearTimeout(timer); + tab.graceTimer = null; + tab.resizeTimer = null; + tab.infoTimer = null; + tab.selectionTimer = null; + tab.pendingOpen = null; + if (tab.metaTimer !== null) clearInterval(tab.metaTimer); + tab.metaTimer = null; + } + + // ---- To the viewers ------------------------------------------------------ + + private broadcast(tab: Tab, message: ServerMessage): void { + for (const viewer of tab.viewers) viewer.send(message); + } + + private queueInfo(tab: Tab): void { + if (tab.viewers.size === 0 || tab.infoTimer !== null) return; + tab.infoTimer = setTimeout(() => { + tab.infoTimer = null; + for (const viewer of tab.viewers) this.sendInfo(tab, viewer); + }, 50); + } + + private sendInfo(tab: Tab, viewer: Viewer): void { + viewer.send({ t: "info", ...tab.info }); + } + + /** Removes upload folders older than an hour. */ + private async sweepUploads(): Promise<void> { + const root = this.seams.uploadDirectory; + await mkdir(root, { recursive: true, mode: 0o700 }).catch(() => undefined); + const names = await readdir(root).catch(() => [] as string[]); + const old = Date.now() - 3_600_000; + for (const name of names) { + const path = join(root, name); + const info = await stat(path).catch(() => null); + if (info !== null && info.mtimeMs < old) + await rm(path, { recursive: true, force: true }).catch(() => undefined); + } + } +} + +/** The targets agent-browser sessions are bound to: its binding files + * (`<socket dir>/<session>.target`, `{"targetId": …}`), as `lazurio browser + * window` reads them (F38 point 3). */ +export async function agentBoundTargets( + socketDirectory: string | undefined, +): Promise<ReadonlySet<string>> { + const bound = new Set<string>(); + if (socketDirectory === undefined) return bound; + const names = await readdir(socketDirectory).catch(() => [] as string[]); + for (const name of names) { + if (!name.endsWith(".target")) continue; + try { + const binding = JSON.parse( + await readFile(join(socketDirectory, name), "utf8"), + ) as { targetId?: unknown }; + if (typeof binding.targetId === "string") bound.add(binding.targetId); + } catch {} + } + return bound; +} diff --git a/src/browser/people/input-map.ts b/src/browser/people/input-map.ts new file mode 100644 index 000000000..2aafb8463 --- /dev/null +++ b/src/browser/people/input-map.ts @@ -0,0 +1,355 @@ +import { + type ClientMessage, + type FrameHeader, + type MouseButton, + modifierBits, + type ViewSize, + viewSizeLimits, +} from "./protocol"; + +/** The person's input as the remote tab takes it (decision F39 point 6): + * pure functions over plain objects. The view page (`client.ts`) applies + * them to its DOM events; the tests run them without a DOM. The remote + * browser runs on Linux, whatever the person's computer is. */ + +export type Point = Readonly<{ x: number; y: number }>; +export type Size = Readonly<{ width: number; height: number }>; +export type Rect = Readonly<{ + left: number; + top: number; + width: number; + height: number; +}>; + +export type KeyMessage = Extract<ClientMessage, { t: "key" }>; + +/** Where an image is drawn inside an area with "contain": as large as fits, + * centred, the rest of the area left as bars. Empty for an empty area or + * image. */ +export function containRect(area: Size, image: Size): Rect { + if ( + !(area.width > 0 && area.height > 0 && image.width > 0 && image.height > 0) + ) + return { left: 0, top: 0, width: 0, height: 0 }; + const scale = Math.min(area.width / image.width, area.height / image.height); + const width = image.width * scale; + const height = image.height * scale; + return { + left: (area.width - width) / 2, + top: (area.height - height) / 2, + width, + height, + }; +} + +/** A pointer position in the person's page as CSS pixels of the remote + * page's viewport, through `drawn`, the rectangle the frame occupies (client + * coordinates). Null outside the frame. The frame shows the remote screen of + * `deviceWidth` × `deviceHeight`, its page starting `offsetTop` down. */ +export function mapPoint( + client: Point, + drawn: Rect, + header: FrameHeader, +): Point | null { + if (!(drawn.width > 0 && drawn.height > 0)) return null; + const x = client.x - drawn.left; + const y = client.y - drawn.top; + if (x < 0 || y < 0 || x > drawn.width || y > drawn.height) return null; + return { + x: (x / drawn.width) * header.deviceWidth, + y: (y / drawn.height) * header.deviceHeight - header.offsetTop, + }; +} + +/** The point moved onto the nearest edge of `drawn`: a drag that leaves the + * frame keeps moving along its edge and is released there. */ +export function clampInto(client: Point, drawn: Rect): Point { + return { + x: Math.min(drawn.left + drawn.width, Math.max(drawn.left, client.x)), + y: Math.min(drawn.top + drawn.height, Math.max(drawn.top, client.y)), + }; +} + +type Modifiers = Readonly<{ + ctrlKey: boolean; + metaKey: boolean; + altKey: boolean; + shiftKey: boolean; +}>; + +/** The modifier bits for the remote browser. On a Mac the person's ⌘ is the + * remote's Ctrl, so ⌘A selects all and ⌘Z undoes there; Ctrl stays Ctrl. */ +export function modifiersOf(event: Modifiers, mac: boolean): number { + return ( + (event.altKey ? modifierBits.alt : 0) | + (event.ctrlKey || (mac && event.metaKey) ? modifierBits.ctrl : 0) | + (!mac && event.metaKey ? modifierBits.meta : 0) | + (event.shiftKey ? modifierBits.shift : 0) + ); +} + +/** A keyboard event of the person's page as `keyAction` reads it. + * `altGraph` is `getModifierState("AltGraph")`. */ +export type KeyInput = Readonly<{ + type: "keydown" | "keyup"; + key: string; + code: string; + keyCode: number; + ctrlKey: boolean; + metaKey: boolean; + altKey: boolean; + shiftKey: boolean; + altGraph: boolean; + location: number; + repeat: boolean; + isComposing: boolean; +}>; + +/** What to do with one key event: the message for the remote tab (null: + * none) and whether the person's browser keeps its own default action. */ +export type KeyAction = Readonly<{ + send: KeyMessage | null; + preventDefault: boolean; +}>; + +const pass: KeyAction = Object.freeze({ send: null, preventDefault: false }); + +/** One key of the person's keyboard for the remote tab. + * + * - A composition (an input method, a phone's keyboard: `keyCode` 229), a + * dead key and an unidentified key are left to the person's browser; their + * text arrives through `compositionend` and `beforeinput`. + * - Paste and copy shortcuts are left to the person's browser too: it fires + * `paste` and `copy`, which carry the person's clipboard and the remote + * selection. Cut is sent (the remote page removes the selection) and also + * left to the browser, which fires `cut` to fill the person's clipboard. + * - `keyCode` is the person's browser's, which is the Windows virtual-key + * code the remote browser wants: `.` stays `.` (190), never Delete (46). + * - Text only on a key down that types: one character without Ctrl or ⌘, + * or with AltGr; Enter types `\r`. */ +export function keyAction(event: KeyInput, mac: boolean): KeyAction { + if (event.isComposing || event.keyCode === 229) return pass; + if (event.key === "Dead" || event.key === "Unidentified") return pass; + // Longer than the protocol takes: no key of a keyboard. + if (event.key.length > 64 || event.code.length > 64) return pass; + const command = + !event.altGraph && + (mac ? event.metaKey && !event.ctrlKey : event.ctrlKey && !event.metaKey); + // The letter, or on a layout without Latin letters its virtual-key code, + // as the person's browser recognises its own shortcuts. + const letter = (char: string, keyCode: number) => + event.key.toLowerCase() === char || event.keyCode === keyCode; + if (command && letter("v", 86)) return pass; + const plain = !event.shiftKey && !event.altKey; + if (command && plain && letter("c", 67)) return pass; + const cut = command && plain && letter("x", 88); + + // The ⌘ key itself is the remote's Control, as its modifier bit is. + const metaAsControl = mac && event.key === "Meta"; + let modifiers = modifiersOf(event, mac); + // AltGr types a character. Windows reports it as Ctrl+Alt, and the Linux + // browser types nothing while Ctrl is held: the remote gets the character + // without them, as AltGr on Linux reports it. + if (event.altGraph) modifiers &= ~(modifierBits.ctrl | modifierBits.alt); + // One character, a single code point: "ě", "@" and an emoji type, a named + // key ("Tab", "ArrowLeft") does not. + const types = + (event.altGraph || (!event.ctrlKey && !event.metaKey)) && + [...event.key].length === 1; + const text = + event.type !== "keydown" + ? undefined + : event.key === "Enter" + ? "\r" + : types + ? event.key + : undefined; + return { + send: { + t: "key", + type: event.type === "keydown" ? "keyDown" : "keyUp", + key: metaAsControl ? "Control" : event.key, + code: metaAsControl + ? event.code.endsWith("Right") + ? "ControlRight" + : "ControlLeft" + : event.code, + keyCode: metaAsControl ? 17 : event.keyCode, + modifiers, + location: event.location, + repeat: event.repeat, + ...(text === undefined ? {} : { text }), + }, + preventDefault: !cut, + }; +} + +/** A key pressed and released with no modifier, for an edit a phone's + * keyboard reports only as `beforeinput`. */ +function press( + key: string, + keyCode: number, + text?: string, +): readonly KeyMessage[] { + const base = { t: "key", key, code: key, keyCode, modifiers: 0 } as const; + return [ + { + ...base, + type: "keyDown", + location: 0, + repeat: false, + ...(text === undefined ? {} : { text }), + }, + { ...base, type: "keyUp", location: 0, repeat: false }, + ]; +} + +/** What a `beforeinput` that no key down carried means for the remote tab + * (a phone's keyboard sends its keys as `keyCode` 229): the typed text, a + * Backspace or an Enter. Null for anything left to the keys' field, such as + * the text of a composition, which `compositionend` sends whole. */ +export function inputAction( + inputType: string, + data: string | null, +): readonly ClientMessage[] | null { + switch (inputType) { + case "insertText": + case "insertReplacementText": + return textMessages(data ?? ""); + case "deleteContentBackward": + return press("Backspace", 8); + case "insertLineBreak": + case "insertParagraph": + return press("Enter", 13, "\r"); + default: + return null; + } +} + +/** Text for the remote caret (a paste, a composition, a phone's word) as + * `text` messages: line breaks as a paste in the page makes them (`\n`), a + * long text in parts that each fit a message, never splitting a + * character. */ +export function textMessages( + text: string, + partLength = 100_000, +): readonly ClientMessage[] { + const normalized = text.replace(/\r\n?/g, "\n"); + const messages: ClientMessage[] = []; + for (let start = 0; start < normalized.length; ) { + let end = Math.min(normalized.length, start + partLength); + const last = normalized.charCodeAt(end - 1); + // The first half of a surrogate pair stays with its second half. + if ( + end < normalized.length && + end - 1 > start && + last >= 0xd800 && + last <= 0xdbff + ) + end -= 1; + messages.push({ t: "text", text: normalized.slice(start, end) }); + start = end; + } + return messages; +} + +/** A wheel's movement in CSS pixels, as the remote page scrolls it: lines of + * 16 pixels, pages of the remote viewport's height. Within the protocol's + * range. */ +export function wheelDelta( + event: Readonly<{ deltaX: number; deltaY: number; deltaMode: number }>, + viewportHeight: number, +): Readonly<{ deltaX: number; deltaY: number }> { + const scale = + event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? viewportHeight : 1; + const clamp = (value: number) => + Math.max(-100_000, Math.min(100_000, value * scale)); + return { deltaX: clamp(event.deltaX), deltaY: clamp(event.deltaY) }; +} + +/** The button of a press or release (DOM `button`). */ +export function mouseButton(button: number): MouseButton { + switch (button) { + case 0: + return "left"; + case 1: + return "middle"; + case 2: + return "right"; + case 3: + return "back"; + case 4: + return "forward"; + default: + return "none"; + } +} + +// DOM `buttons` bits, which CDP counts the same way. +const buttonBits: readonly (readonly [number, MouseButton])[] = [ + [1, "left"], + [4, "middle"], + [2, "right"], + [8, "back"], + [16, "forward"], +]; + +/** The button a move drags with (DOM `buttons`): a remote drag, a text + * selection, needs it on every move, as a native move has it. */ +export function heldButton(buttons: number): MouseButton { + for (const [bit, button] of buttonBits) if (buttons & bit) return button; + return "none"; +} + +/** The `buttons` bit of a DOM `button`; 0 for none. */ +export function buttonBit(button: number): number { + const name = mouseButton(button); + return buttonBits.find(([, held]) => held === name)?.[0] ?? 0; +} + +/** A press of the person's pointer, for counting double and triple clicks: + * a pointer event's `detail` is always 0, so the page counts them itself. */ +export type Press = Readonly<{ + time: number; + x: number; + y: number; + button: MouseButton; + count: number; +}>; + +const multiClickMs = 500; +const multiClickDistance = 4; + +/** The click count of a press: one more than the press before when it was + * the same button, within half a second and four pixels; else 1. */ +export function clickCount( + previous: Press | null, + next: Omit<Press, "count">, +): number { + if ( + previous === null || + previous.button !== next.button || + next.time - previous.time > multiClickMs || + Math.abs(next.x - previous.x) > multiClickDistance || + Math.abs(next.y - previous.y) > multiClickDistance + ) + return 1; + return Math.min(previous.count + 1, 10); +} + +/** The page area's size as `hello` and `size` carry it: whole CSS pixels + * within the protocol's limits, the device pixel ratio from 1 to the + * largest the service renders. */ +export function viewSize(width: number, height: number, dpr: number): ViewSize { + const within = (value: number, min: number, max: number) => + Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : min; + return { + width: Math.round( + within(width, viewSizeLimits.minWidth, viewSizeLimits.maxWidth), + ), + height: Math.round( + within(height, viewSizeLimits.minHeight, viewSizeLimits.maxHeight), + ), + dpr: Math.round(within(dpr, 1, viewSizeLimits.maxDpr) * 100) / 100, + }; +} diff --git a/src/browser/people/page.ts b/src/browser/people/page.ts new file mode 100644 index 000000000..21d786dd7 --- /dev/null +++ b/src/browser/people/page.ts @@ -0,0 +1,169 @@ +/** The page of the people's view (decision F39 point 1), one document for + * `/` and for `/t/<id>`: a thin bar (back, forward, reload and the address), + * a row for a page's new tab, and the page area. The script + * (`/assets/view.js`, `client.ts`) does the rest: it opens a remote tab for + * `/`, fills in the words in the person's language and draws the remote tab. + * No inline script and nothing from another origin: the service's policy is + * `script-src 'self'` and allows only its own styles and images. */ + +// Icons drawn for this page, in the current colour, hidden from assistive +// technology (each button names itself). +const icon = (paths: string, className = "") => + `<svg${className === "" ? "" : ` class="${className}"`} viewBox="0 0 24 24" width="18" height="18" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">${paths}</svg>`; + +const cross = '<path d="M7 7l10 10M17 7L7 17"/>'; + +// The page's icon until the remote page has one of its own (a globe). +const defaultIcon = + "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%2380808a' stroke-width='1.6'%3E%3Ccircle cx='12' cy='12' r='9'/%3E%3Cpath d='M3 12h18M12 3c2.4 2.5 3.6 5.5 3.6 9s-1.2 6.5-3.6 9c-2.4-2.5-3.6-5.5-3.6-9s1.2-6.5 3.6-9z'/%3E%3C/svg%3E"; + +const styles = ` +*, *::before, *::after { box-sizing: border-box; } +[hidden] { display: none !important; } +:root { + color-scheme: light dark; + --chrome: #f3f3f4; --line: #dcdcdf; --text: #1d1d1f; --muted: #6e6e73; + --field: #ffffff; --hover: rgba(0, 0, 0, 0.07); --accent: #2563eb; + --on-accent: #ffffff; --page: #e9e9eb; --panel: #ffffff; --notice: #eef2fb; + --veil: rgba(233, 233, 235, 0.92); --error: #c5221f; + --shadow: 0 8px 28px rgba(0, 0, 0, 0.16), 0 1px 3px rgba(0, 0, 0, 0.1); +} +@media (prefers-color-scheme: dark) { + :root { + --chrome: #2a2a2d; --line: #3d3d42; --text: #ececef; --muted: #a1a1a8; + --field: #1c1c1e; --hover: rgba(255, 255, 255, 0.09); --accent: #7aa2ff; + --on-accent: #111114; --page: #18181a; --panel: #2f2f33; --notice: #26304a; + --veil: rgba(24, 24, 26, 0.9); --error: #f28b82; + --shadow: 0 8px 28px rgba(0, 0, 0, 0.5); + } +} +html, body { height: 100%; margin: 0; } +body { + overflow: hidden; overscroll-behavior: none; background: var(--page); + color: var(--text); -webkit-text-size-adjust: 100%; + font: 13px/1.35 system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; +} +.view { position: fixed; inset: 0; display: flex; flex-direction: column; } +.bar { + flex: none; display: flex; align-items: center; gap: 2px; height: 40px; + padding: 0 6px; background: var(--chrome); border-bottom: 1px solid var(--line); +} +.tool { + flex: none; display: inline-grid; place-items: center; width: 30px; height: 30px; + padding: 0; border: 0; border-radius: 50%; background: transparent; + color: var(--text); cursor: pointer; +} +.tool:hover:not(:disabled) { background: var(--hover); } +.tool:disabled { opacity: 0.35; cursor: default; } +.tool:focus-visible, .button:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } +#reload[data-state="reload"] .stop-icon, #reload[data-state="stop"] .reload-icon { display: none; } +.address { + flex: 1; min-width: 0; height: 30px; margin-left: 4px; padding: 0 12px; + border: 1px solid var(--line); border-radius: 15px; background: var(--field); + color: var(--text); font: inherit; font-size: 14px; outline: none; +} +.address:focus { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); } +.address::placeholder { color: var(--muted); } +.address:disabled { opacity: 0.6; } +.notice { + flex: none; display: flex; align-items: center; gap: 8px; min-height: 36px; + padding: 3px 4px 3px 12px; background: var(--notice); border-bottom: 1px solid var(--line); +} +.notice-text { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.area { + position: relative; flex: 1; min-height: 0; overflow: hidden; background: var(--page); + touch-action: none; user-select: none; -webkit-user-select: none; + -webkit-touch-callout: none; -webkit-tap-highlight-color: transparent; +} +.page { position: absolute; inset: 0; overflow: hidden; } +.screen { position: absolute; inset: 0; display: block; width: 100%; height: 100%; touch-action: none; } +.keys { + position: absolute; left: 0; top: 0; width: 1px; height: 1px; margin: 0; padding: 0; + border: 0; opacity: 0; resize: none; overflow: hidden; white-space: pre; + color: transparent; background: transparent; caret-color: transparent; + font-size: 16px; pointer-events: none; +} +.status, .banner { + position: absolute; left: 50%; max-width: calc(100% - 24px); overflow: hidden; + text-overflow: ellipsis; white-space: nowrap; border-radius: 999px; + background: var(--panel); box-shadow: var(--shadow); pointer-events: none; +} +.status { top: 50%; transform: translate(-50%, -50%); padding: 7px 14px; color: var(--muted); } +.banner { z-index: 4; top: 10px; transform: translateX(-50%); padding: 6px 12px; font-size: 12px; } +.popups { position: absolute; inset: 0; z-index: 2; display: grid; place-items: center; pointer-events: none; } +.popup { + grid-area: 1 / 1; display: flex; flex-direction: column; overflow: hidden; + border: 1px solid var(--line); border-radius: 10px; background: var(--panel); + box-shadow: var(--shadow); pointer-events: auto; +} +.popup-head { + flex: none; display: flex; align-items: center; gap: 6px; height: 34px; + padding: 0 2px 0 12px; background: var(--chrome); border-bottom: 1px solid var(--line); +} +.popup-title { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--muted); } +.popup-body { position: relative; flex: none; width: 480px; height: 600px; background: var(--page); } +.end { + position: absolute; inset: 0; z-index: 3; display: flex; flex-direction: column; + align-items: center; justify-content: center; gap: 14px; padding: 24px; + text-align: center; background: var(--veil); +} +.end p { margin: 0; font-size: 15px; } +.button { + height: 30px; padding: 0 14px; border: 1px solid var(--line); border-radius: 8px; + background: var(--field); color: var(--text); font: inherit; cursor: pointer; white-space: nowrap; +} +.button.primary { border-color: var(--accent); background: var(--accent); color: var(--on-accent); } +.button:disabled { opacity: 0.55; cursor: default; } +.modal { + width: min(420px, calc(100vw - 32px)); padding: 18px; border: 1px solid var(--line); + border-radius: 12px; background: var(--panel); color: var(--text); box-shadow: var(--shadow); +} +.modal::backdrop { background: rgba(0, 0, 0, 0.32); } +.modal h2 { margin: 0 0 10px; font-size: 15px; font-weight: 600; } +.modal .message { margin: 0 0 14px; max-height: 50vh; overflow: auto; white-space: pre-wrap; overflow-wrap: anywhere; } +.modal .field { + display: block; width: 100%; height: 32px; margin: 0 0 14px; padding: 0 10px; + border: 1px solid var(--line); border-radius: 8px; background: var(--field); + color: var(--text); font: inherit; font-size: 16px; +} +.modal input[type="file"] { display: block; width: 100%; margin: 0 0 14px; } +.modal .error { margin: 0 0 12px; color: var(--error); } +.actions { display: flex; justify-content: flex-end; gap: 8px; } +`; + +const page = `<!doctype html> +<html> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1, interactive-widget=resizes-content"> +<meta name="color-scheme" content="light dark"> +<title> + + + + +
+
+ + + + +
+ +
+ +
+
+ + + +`; + +/** The complete document of the view page. */ +export function viewPage(): string { + return page; +} diff --git a/src/browser/people/protocol.ts b/src/browser/people/protocol.ts new file mode 100644 index 000000000..87bb80b2d --- /dev/null +++ b/src/browser/people/protocol.ts @@ -0,0 +1,321 @@ +/** The people's view of the Environment browser (decision F39): the messages + * between the view page in a person's tab and the view service on the + * Environment, over `wss://browser.<…>/t//live`. One WebSocket shows one + * remote tab. Text messages are JSON objects with a `t` member; frames are + * binary (`encodeFrame`). Shared by the service and the page, so both sides + * read one definition. */ + +/** A DevTools target id as Chrome writes it: 32 hexadecimal digits. The view + * takes nothing else as ``. */ +export const isTargetId = (value: unknown): value is string => + typeof value === "string" && /^[0-9A-F]{32}$/i.test(value); + +/** The page area of the view in CSS pixels and its device pixel ratio. */ +export type ViewSize = Readonly<{ width: number; height: number; dpr: number }>; + +export const viewSizeLimits = Object.freeze({ + minWidth: 200, + minHeight: 150, + maxWidth: 3840, + maxHeight: 2400, + maxDpr: 3, +}); + +/** Modifier bits as CDP's `Input` domain counts them. */ +export const modifierBits = Object.freeze({ + alt: 1, + ctrl: 2, + meta: 4, + shift: 8, +}); + +export type MouseButton = + | "none" + | "left" + | "middle" + | "right" + | "back" + | "forward"; + +export type TouchPoint = Readonly<{ x: number; y: number; id: number }>; + +/** From the page to the service. Coordinates are CSS pixels of the remote + * page's viewport, already mapped by the page from the frame it shows. */ +export type ClientMessage = + /** First message: the page area. `resize` false keeps the window's size + * (a pop-up shown over the page keeps the size its opener asked for). */ + | Readonly<{ + t: "hello"; + width: number; + height: number; + dpr: number; + resize: boolean; + }> + | Readonly<{ t: "size"; width: number; height: number; dpr: number }> + | Readonly<{ + t: "mouse"; + type: "mousePressed" | "mouseReleased" | "mouseMoved" | "mouseWheel"; + x: number; + y: number; + button: MouseButton; + buttons: number; + clickCount: number; + modifiers: number; + deltaX?: number; + deltaY?: number; + }> + /** A key as the person's browser reported it. `keyCode` is the DOM + * `keyCode`, which is the Windows virtual-key code CDP wants. `text` only + * for a key that types it (no Ctrl or Meta). */ + | Readonly<{ + t: "key"; + type: "keyDown" | "keyUp"; + key: string; + code: string; + keyCode: number; + text?: string; + modifiers: number; + location: number; + repeat: boolean; + }> + /** Text to insert at the caret: a paste or a finished composition. */ + | Readonly<{ t: "text"; text: string }> + | Readonly<{ + t: "touch"; + type: "touchStart" | "touchMove" | "touchEnd" | "touchCancel"; + points: readonly TouchPoint[]; + modifiers: number; + }> + | Readonly<{ t: "nav"; action: "back" | "forward" | "reload" | "stop" }> + | Readonly<{ t: "goto"; url: string }> + | Readonly<{ t: "dialog"; accept: boolean; text?: string }> + /** The person dismissed the file chooser without picking a file. */ + | Readonly<{ t: "files-cancel"; token: string }> + /** Close this remote tab (a pop-up's close button). */ + | Readonly<{ t: "close" }>; + +/** From the service to the page (text messages; frames are binary). */ +export type ServerMessage = + /** Where the tab is: its address, title, icon and history. */ + | Readonly<{ + t: "info"; + url: string; + title: string; + favicon: string | null; + canGoBack: boolean; + canGoForward: boolean; + loading: boolean; + }> + /** The remote selection, for the person's copy and cut. */ + | Readonly<{ t: "selection"; text: string }> + /** The tab opened a pop-up (it can reach its opener): show it over the + * page through its own view, `/t/`. */ + | Readonly<{ t: "popup"; id: string; url: string }> + /** The tab opened a new tab (no opener access): offer it. */ + | Readonly<{ t: "new-tab"; id: string; url: string }> + | Readonly<{ + t: "dialog"; + kind: "alert" | "confirm" | "prompt" | "beforeunload"; + message: string; + defaultPrompt: string; + }> + | Readonly<{ t: "dialog-closed" }> + /** The page asked for files: upload them with `POST /t//files?token=`. */ + | Readonly<{ t: "files"; multiple: boolean; token: string }> + | Readonly<{ t: "files-closed"; token: string }> + /** The tab no longer exists. The socket closes after this. */ + | Readonly<{ t: "closed" }> + /** The service lost the browser and reconnects, or found it again. */ + | Readonly<{ t: "browser"; state: "connected" | "reconnecting" }>; + +/** What a frame says about itself: its size in device pixels and the page's + * viewport in CSS pixels it shows (CDP `ScreencastFrameMetadata`). */ +export type FrameHeader = Readonly<{ + /** Width and height of the image, device pixels. */ + width: number; + height: number; + /** The page's viewport, CSS pixels. */ + deviceWidth: number; + deviceHeight: number; + /** Top of the page in the image, CSS pixels (non-zero only with an + * emulated mobile top bar). */ + offsetTop: number; + pageScaleFactor: number; +}>; + +/** One binary message per frame: a 4-byte big-endian length, the header as + * UTF-8 JSON, then the JPEG. */ +export function encodeFrame(header: FrameHeader, jpeg: Uint8Array): Uint8Array { + const head = new TextEncoder().encode(JSON.stringify(header)); + const out = new Uint8Array(4 + head.length + jpeg.length); + new DataView(out.buffer).setUint32(0, head.length, false); + out.set(head, 4); + out.set(jpeg, 4 + head.length); + return out; +} + +export function decodeFrame( + data: ArrayBuffer | Uint8Array, +): { header: FrameHeader; jpeg: Uint8Array } | null { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + if (bytes.length < 4) return null; + const length = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength, + ).getUint32(0, false); + if (length > 4096 || 4 + length > bytes.length) return null; + try { + const header = JSON.parse( + new TextDecoder().decode(bytes.subarray(4, 4 + length)), + ) as FrameHeader; + const numbers = [ + header.width, + header.height, + header.deviceWidth, + header.deviceHeight, + header.offsetTop, + header.pageScaleFactor, + ]; + if (!numbers.every((n) => typeof n === "number" && Number.isFinite(n))) + return null; + return { header, jpeg: bytes.subarray(4 + length) }; + } catch { + return null; + } +} + +const finite = (value: unknown, min: number, max: number) => + typeof value === "number" && + Number.isFinite(value) && + value >= min && + value <= max; + +const coordinate = (value: unknown) => finite(value, -100_000, 100_000); +const small = (value: unknown) => finite(value, 0, 0xffff); +const text = (value: unknown, max: number) => + typeof value === "string" && value.length <= max; + +const mouseTypes = new Set([ + "mousePressed", + "mouseReleased", + "mouseMoved", + "mouseWheel", +]); +const buttons = new Set(["none", "left", "middle", "right", "back", "forward"]); +const touchTypes = new Set([ + "touchStart", + "touchMove", + "touchEnd", + "touchCancel", +]); + +/** Reads one text message of the page; null for anything the protocol does + * not have. The service acts only on what this returns. */ +export function parseClientMessage(raw: string): ClientMessage | null { + if (raw.length > 1_100_000) return null; + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + return null; + } + if (typeof value !== "object" || value === null || Array.isArray(value)) + return null; + const m = value as Record; + const size = () => + finite(m.width, 1, 100_000) && + finite(m.height, 1, 100_000) && + finite(m.dpr, 0.1, 10); + switch (m.t) { + case "hello": + return size() && typeof m.resize === "boolean" + ? (m as ClientMessage) + : null; + case "size": + return size() ? (m as ClientMessage) : null; + case "mouse": + return mouseTypes.has(m.type as string) && + coordinate(m.x) && + coordinate(m.y) && + buttons.has(m.button as string) && + small(m.buttons) && + finite(m.clickCount, 0, 10) && + small(m.modifiers) && + (m.deltaX === undefined || coordinate(m.deltaX)) && + (m.deltaY === undefined || coordinate(m.deltaY)) + ? (m as ClientMessage) + : null; + case "key": + return (m.type === "keyDown" || m.type === "keyUp") && + text(m.key, 64) && + text(m.code, 64) && + small(m.keyCode) && + (m.text === undefined || (text(m.text, 16) && m.text !== "")) && + small(m.modifiers) && + finite(m.location, 0, 3) && + typeof m.repeat === "boolean" + ? (m as ClientMessage) + : null; + case "text": + return text(m.text, 1_000_000) && m.text !== "" + ? (m as ClientMessage) + : null; + case "touch": + return touchTypes.has(m.type as string) && + Array.isArray(m.points) && + m.points.length <= 10 && + m.points.every( + (p: unknown) => + typeof p === "object" && + p !== null && + coordinate((p as TouchPoint).x) && + coordinate((p as TouchPoint).y) && + finite((p as TouchPoint).id, 0, 1_000_000), + ) && + small(m.modifiers) + ? (m as ClientMessage) + : null; + case "nav": + return ["back", "forward", "reload", "stop"].includes(m.action as string) + ? (m as ClientMessage) + : null; + case "goto": + return text(m.url, 8192) && m.url !== "" ? (m as ClientMessage) : null; + case "dialog": + return typeof m.accept === "boolean" && + (m.text === undefined || text(m.text, 100_000)) + ? (m as ClientMessage) + : null; + case "files-cancel": + return text(m.token, 64) ? (m as ClientMessage) : null; + case "close": + return m as ClientMessage; + default: + return null; + } +} + +/** What the address bar's text means: an address, or a search. A bare + * host (`example.com`, `localhost:3000`) gets a scheme; anything with a + * space or without a dot is a search. Never a `javascript:` or `file:` + * address: only http, https and about:blank are navigated to. */ +export function addressOf(input: string): string { + const value = input.trim(); + if (value === "about:blank") return value; + try { + const url = new URL(value); + if (url.protocol === "http:" || url.protocol === "https:") return url.href; + } catch {} + if (!/\s/.test(value)) { + const local = /^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?(\/.*)?$/i.test( + value, + ); + if (local || /^[^/?#]+\.[a-z]{2,}(:\d+)?([/?#].*)?$/i.test(value)) { + try { + return new URL(`${local ? "http" : "https"}://${value}`).href; + } catch {} + } + } + return `https://www.google.com/search?q=${encodeURIComponent(value)}`; +} diff --git a/src/browser/people/serve.ts b/src/browser/people/serve.ts new file mode 100644 index 000000000..a23476f57 --- /dev/null +++ b/src/browser/people/serve.ts @@ -0,0 +1,72 @@ +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { browserCdpPort, browserScreenSize } from "../units"; +import { agentBrowserSocketDirectory } from "../window"; +import { agentBoundTargets } from "./hub"; +import { loopbackBrowser, startViewService } from "./service"; + +/** `lazurio browser serve --port --origin `: the people's view + * service (decision F39), as `lazurio-browser-view.service` runs it. It runs + * until it is stopped. Not for agents: they use `lazurio browser window`. */ +export const serveSynopsis = + "browser serve --port --origin "; + +export function parseServeArgs( + args: readonly string[], +): { port: number; origin: string } | null { + try { + const { values, positionals } = parseArgs({ + args: [...args], + strict: true, + allowPositionals: true, + options: { port: { type: "string" }, origin: { type: "string" } }, + }); + if (positionals.length !== 0) return null; + const port = Number(values.port); + if (!Number.isInteger(port) || port < 1 || port > 65535) return null; + const origin = new URL(values.origin ?? ""); + if ( + (origin.protocol !== "https:" && origin.protocol !== "http:") || + origin.origin !== values.origin + ) + return null; + return { port, origin: origin.origin }; + } catch { + return null; + } +} + +export async function runViewService( + args: readonly string[], + env: Readonly>, +): Promise { + const options = parseServeArgs(args); + if (options === null) { + console.error(`Usage: lazurio ${serveSynopsis}`); + return 2; + } + const log = (line: string) => console.error(line); + const runtime = env.XDG_RUNTIME_DIR; + const service = startViewService({ + origin: options.origin, + port: options.port, + log, + hub: { + connect: loopbackBrowser(browserCdpPort), + boundTargets: () => agentBoundTargets(agentBrowserSocketDirectory(env)), + uploadDirectory: join( + runtime?.startsWith("/") ? runtime : tmpdir(), + "lazurio-browser-uploads", + ), + screen: browserScreenSize, + log, + }, + }); + await new Promise((resolve) => { + process.once("SIGTERM", () => resolve()); + process.once("SIGINT", () => resolve()); + }); + await service.stop(); + return 0; +} diff --git a/src/browser/people/service.ts b/src/browser/people/service.ts new file mode 100644 index 000000000..c554c5d24 --- /dev/null +++ b/src/browser/people/service.ts @@ -0,0 +1,300 @@ +import type { Server, ServerWebSocket } from "bun"; +import { viewScript } from "./bundle" with { type: "macro" }; +import { browserSocketUrl, type CdpConnection, connectCdp } from "./cdp"; +import { BrowserHub, type HubSeams, type Viewer } from "./hub"; +import { viewPage } from "./page"; +import { parseClientMessage } from "./protocol"; + +/** The people's view service (decision F39): `lazurio browser serve` behind + * the gateway's `browser.` route. It serves the view page, opens remote tabs + * and carries one WebSocket per person's tab. The gateway's session is the + * only admission (F39 point 8): this service checks that every request names + * its own host and that every socket and every change comes from its own + * origin, and it listens on loopback only. */ + +export type ViewServiceOptions = Readonly<{ + /** The view's public origin, `https://browser.<…>`, from the handover. */ + origin: string; + /** The loopback port the gateway forwards to. */ + port: number; + hub: HubSeams; + log: (line: string) => void; +}>; + +type SocketData = { targetId: string; viewer: Viewer | null }; + +const script = viewScript(); +const maxUploadBytes = 256 * 1024 * 1024; + +/** `frame-ancestors` for the view: the Environment's own origins, the + * siblings of the view's host (`https://*...lazurio.io`), never + * another Environment. A host with fewer than three labels (a test on + * localhost) is framed by itself only. */ +export function frameAncestors(origin: string): string { + const host = new URL(origin).hostname; + const labels = host.split("."); + return labels.length >= 3 + ? `'self' https://*.${labels.slice(1).join(".")}` + : "'self'"; +} + +export function pageHeaders(origin: string): Record { + const url = new URL(origin); + const socket = `${url.protocol === "https:" ? "wss" : "ws"}://${url.host}`; + return { + "content-type": "text/html; charset=utf-8", + "cache-control": "no-store", + "content-security-policy": [ + "default-src 'none'", + "script-src 'self'", + "style-src 'unsafe-inline'", + "img-src 'self' https: data: blob:", + `connect-src 'self' ${socket}`, + "base-uri 'none'", + "form-action 'none'", + `frame-ancestors ${frameAncestors(origin)}`, + ].join("; "), + "x-content-type-options": "nosniff", + "referrer-policy": "no-referrer", + }; +} + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { + "content-type": "application/json", + "cache-control": "no-store", + "x-content-type-options": "nosniff", + }, + }); + +const livePath = /^\/t\/([0-9A-Fa-f]{32})\/live$/; +const filesPath = /^\/t\/([0-9A-Fa-f]{32})\/files$/; +const viewPath = /^\/t\/([0-9A-Fa-f]{32})$/; + +export type RequestDecision = + | Readonly<{ kind: "health" }> + | Readonly<{ kind: "page" }> + | Readonly<{ kind: "script" }> + | Readonly<{ kind: "create" }> + | Readonly<{ kind: "files"; targetId: string }> + | Readonly<{ kind: "live"; targetId: string }> + | Readonly<{ kind: "refused"; status: number; error: string }>; + +/** What a request is, after the Host and Origin checks (F39 point 8). + * Pure, so the checks are tested without a socket. */ +export function classifyRequest( + request: Readonly<{ + method: string; + url: string; + host: string | null; + origin: string | null; + upgrade: boolean; + }>, + viewOrigin: string, + port: number, +): RequestDecision { + const own = new URL(viewOrigin); + const path = new URL(request.url, "http://view.invalid").pathname; + const loopback = + request.host === `127.0.0.1:${port}` || + request.host === `localhost:${port}`; + if (path === "/.lazurio/health" && request.method === "GET") + return request.host === own.host || loopback + ? { kind: "health" } + : { kind: "refused", status: 421, error: "wrong-host" }; + if (request.host !== own.host) + return { kind: "refused", status: 421, error: "wrong-host" }; + const sameOrigin = request.origin === own.origin; + const live = livePath.exec(path); + if (live !== null) { + if (request.method !== "GET" || !request.upgrade) + return { kind: "refused", status: 426, error: "upgrade-required" }; + if (!sameOrigin) + return { kind: "refused", status: 403, error: "wrong-origin" }; + return { kind: "live", targetId: (live[1] as string).toUpperCase() }; + } + if (path === "/api/tabs") { + if (request.method !== "POST") + return { kind: "refused", status: 405, error: "method-not-allowed" }; + if (!sameOrigin) + return { kind: "refused", status: 403, error: "wrong-origin" }; + return { kind: "create" }; + } + const files = filesPath.exec(path); + if (files !== null) { + if (request.method !== "POST") + return { kind: "refused", status: 405, error: "method-not-allowed" }; + if (!sameOrigin) + return { kind: "refused", status: 403, error: "wrong-origin" }; + return { kind: "files", targetId: (files[1] as string).toUpperCase() }; + } + if (request.method !== "GET" && request.method !== "HEAD") + return { kind: "refused", status: 405, error: "method-not-allowed" }; + if (path === "/" || viewPath.test(path)) return { kind: "page" }; + if (path === "/assets/view.js") return { kind: "script" }; + return { kind: "refused", status: 404, error: "not-found" }; +} + +export type ViewService = Readonly<{ + server: Server; + hub: BrowserHub; + stop: () => Promise; +}>; + +export function startViewService(options: ViewServiceOptions): ViewService { + const hub = new BrowserHub(options.hub); + const headers = pageHeaders(options.origin); + const page = viewPage(); + const server = Bun.serve({ + hostname: "127.0.0.1", + port: options.port, + maxRequestBodySize: maxUploadBytes, + async fetch(request, server) { + const decision = classifyRequest( + { + method: request.method, + url: request.url, + host: request.headers.get("host"), + origin: request.headers.get("origin"), + upgrade: + request.headers.get("upgrade")?.toLowerCase() === "websocket", + }, + options.origin, + server.port ?? options.port, + ); + switch (decision.kind) { + case "refused": + return json({ error: decision.error }, decision.status); + case "health": + return json( + { + ok: hub.connected, + browser: hub.connected ? "connected" : "unreachable", + }, + hub.connected ? 200 : 503, + ); + case "page": + return new Response(page, { headers }); + case "script": + return new Response(script, { + headers: { + "content-type": "text/javascript; charset=utf-8", + "cache-control": "no-cache", + "x-content-type-options": "nosniff", + }, + }); + case "create": { + let size: { width: number; height: number; dpr: number } | null = + null; + try { + const body = (await request.json()) as Record; + if ( + [body.width, body.height, body.dpr].every( + (n) => typeof n === "number" && Number.isFinite(n) && n > 0, + ) + ) + size = { + width: body.width as number, + height: body.height as number, + dpr: body.dpr as number, + }; + } catch {} + const id = await hub.createTab(size); + return id === null + ? json({ error: "browser-unreachable" }, 503) + : json({ id, view: `/t/${id}` }, 201); + } + case "files": { + const token = new URL(request.url).searchParams.get("token") ?? ""; + if (!/^[0-9a-f]{24}$/.test(token)) + return json({ error: "invalid-request" }, 400); + let files: File[]; + try { + const form = await request.formData(); + files = form + .getAll("file") + .filter((value): value is File => value instanceof File); + } catch { + return json({ error: "invalid-request" }, 400); + } + const done = await hub + .upload(decision.targetId, token, files) + .catch(() => false); + return done + ? new Response(null, { status: 204 }) + : json({ error: "not-accepted" }, 409); + } + case "live": { + if (!hub.hasTab(decision.targetId)) + return json({ error: "no-such-tab" }, 404); + const upgraded = server.upgrade(request, { + data: { targetId: decision.targetId, viewer: null }, + }); + return upgraded ? undefined : json({ error: "upgrade-failed" }, 400); + } + } + }, + websocket: { + perMessageDeflate: false, + maxPayloadLength: 2 * 1024 * 1024, + backpressureLimit: 64 * 1024 * 1024, + closeOnBackpressureLimit: true, + open(ws: ServerWebSocket) { + const viewer: Viewer = { + send: (message) => { + ws.send(JSON.stringify(message)); + }, + sendFrame: (frame) => { + ws.send(frame); + }, + buffered: () => ws.getBufferedAmount(), + close: () => ws.close(1000, "closed"), + size: null, + resize: true, + }; + ws.data.viewer = viewer; + if (!hub.attach(ws.data.targetId, viewer)) { + viewer.send({ t: "closed" }); + ws.close(1000, "closed"); + } + }, + message(ws: ServerWebSocket, data) { + const viewer = ws.data.viewer; + if (viewer === null || typeof data !== "string") return; + const message = parseClientMessage(data); + if (message === null) return; + void hub.handle(ws.data.targetId, viewer, message).catch((error) => { + options.log(`message failed: ${String(error)}`); + }); + }, + drain(ws: ServerWebSocket) { + hub.drained(ws.data.targetId); + }, + close(ws: ServerWebSocket) { + const viewer = ws.data.viewer; + if (viewer !== null) hub.detach(ws.data.targetId, viewer); + }, + }, + }); + hub.start(); + options.log( + `people's view of the Environment browser on 127.0.0.1:${server.port} for ${options.origin}`, + ); + return Object.freeze({ + server, + hub, + stop: async () => { + hub.stop(); + await server.stop(true); + }, + }); +} + +/** The production connection: the browser target on the loopback DevTools + * port, asked again on every reconnect (a restarted browser has a new id). */ +export const loopbackBrowser = + (cdpPort: number): (() => Promise) => + async () => + connectCdp(await browserSocketUrl(cdpPort)); diff --git a/src/browser/units.ts b/src/browser/units.ts index d6ec088f3..cd6a1bf28 100644 --- a/src/browser/units.ts +++ b/src/browser/units.ts @@ -1,6 +1,7 @@ import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { writeDurableFile } from "../update/durable-file"; +import { layout } from "../update/layout"; import type { ProcessRunner } from "../update/self-check"; import { detectServiceControl, @@ -12,16 +13,22 @@ import { unitPath, userUnitDirectory, } from "../update/service-control"; +import { + browserExtensionDigest, + writeBrowserExtension, +} from "./people/extension"; /** The Environment browser of a Remote Environment (root decision 0191, - * decision F38): one virtual screen, one Chromium with one persistent profile - * that every thread, agent and bot of the Environment shares, and the - * agent-browser dashboard as the person's view of its windows. Three - * installer units, converged like the Codex app-server unit (F29): written - * when their text differs, enabled and started; `install` and `update` never - * stop the screen or the browser, since that would close every agent's - * window. Machines delivers the packages (Xvfb, Chrome for Testing through - * agent-browser) and the gateway route; the Platform owns what runs. */ + * decisions F38 and F39): one virtual screen, one Chromium with one + * persistent profile that every thread, agent and bot of the Environment + * shares, and the people's view, where one tab of a person is one remote + * tab. Three installer units, converged like the Codex app-server unit (F29): + * written when their text differs, enabled and started. `install` and + * `update` never stop the screen; they restart the browser only when its own + * unit changed (a new flag or a new extension, F39 point 10), since that + * closes every agent's window. Machines delivers the packages (Xvfb, Chrome + * for Testing through agent-browser) and the gateway route; the Platform + * owns what runs. */ export const displayUnit = "lazurio-display.service"; export const browserUnit = "lazurio-browser.service"; export const browserViewUnit = "lazurio-browser-view.service"; @@ -50,13 +57,16 @@ export const browserCdpPort = 9222; export const browserProfileDirectory = (home: string) => join(home, ".local", "share", "lazurio-browser", "profile"); -/** The size of a thread's window and of its view, so the view's pointer - * lands where the page is (the dashboard maps clicks with this viewport). */ +/** The size of a thread's window until a person's view resizes it (F39 + * point 3). */ export const browserWindowSize = Object.freeze({ width: 1280, height: 800 }); +/** The virtual screen; no window grows past it. */ +export const browserScreenSize = Object.freeze({ width: 1920, height: 1080 }); + /** What the gateway tells the Environment about the view, from the handover * (`entry.browser`, Machines): where the person reaches it and the loopback - * port the dashboard listens on. Never derived by convention. */ + * port the view service listens on. Never derived by convention. */ export type BrowserEntry = Readonly<{ origin: string; listenPort: number }>; // Chrome for Testing as agent-browser installs it under the operator's home @@ -70,14 +80,17 @@ const chromeCandidates = [ "%h/.agent-browser/browsers/chrome-*/chrome-linux64/chrome", ]; -/** Chrome's flags, one per line of the docs (decision F38): +/** Chrome's flags, one per line of the docs (decisions F38 and F39): * - the persistent profile and loopback DevTools; * - no first-run or default-browser questions, nobody answers them; * - `--password-store=basic`: no desktop keyring on a virtual screen; * - no background throttling: an agent's window keeps working while the * person watches another one; * - no crash-restore bubble after a restart of the unit; - * - the first window at a thread window's size. */ + * - the first window at a thread window's size; + * - permission prompts are denied, never shown where nobody sees them; + * - the Lazurio extension (a window per tab, passkeys declined), with the + * feature that turns `--load-extension` off in branded builds disabled. */ export const browserFlags = Object.freeze([ "--user-data-dir=%h/.local/share/lazurio-browser/profile", `--remote-debugging-port=${browserCdpPort}`, @@ -89,6 +102,9 @@ export const browserFlags = Object.freeze([ "--disable-backgrounding-occluded-windows", "--hide-crash-restore-bubble", `--window-size=${browserWindowSize.width},${browserWindowSize.height + 100}`, + "--deny-permission-prompts", + "--load-extension=%h/.local/share/lazurio-browser/extension", + "--disable-features=DisableLoadExtensionCommandLineSwitch", ]); /** The screen: Xvfb on `:1`, no TCP. `Type=simple` and a start that waits @@ -116,7 +132,8 @@ export function renderDisplayUnit(): string { /** The browser: the newest Chrome for Testing, headed on the screen, `exec` * so Chrome is the unit's main process. Bound to the screen; restarted * whenever it ends (a person closing the last window included), except when - * there is no Chrome to run. */ + * there is no Chrome to run. The extension's digest is part of the text, so + * a new extension is a changed unit, which restarts the browser. */ export function renderBrowserUnit(): string { const command = [ `b=$$(ls -d ${chromeCandidates.join(" ")} 2>/dev/null | sort -V | tail -n 1)`, @@ -133,6 +150,7 @@ export function renderBrowserUnit(): string { "[Service]", "Type=simple", `Environment=DISPLAY=${browserDisplay}`, + `Environment=LAZURIO_BROWSER_EXTENSION=${browserExtensionDigest}`, `ExecStart=/bin/sh -c '${command}'`, "Restart=always", "RestartSec=3", @@ -144,28 +162,35 @@ export function renderBrowserUnit(): string { ].join("\n"); } -/** The view: agent-browser's own dashboard, allowed for the gateway's origin - * only, so its access token and its Origin check stay on (decision F38). - * `dashboard start` detaches and exits, as Codex's daemon does in F29: - * `oneshot`, `RemainAfterExit=yes`, `KillMode=process`, stopped by agent- - * browser itself. `AGENT_BROWSER_CDP` makes what the dashboard runs use the - * Environment browser, never a browser of its own. */ -export function renderBrowserViewUnit(entry: BrowserEntry): string { +/** The view: the Platform's own people's view (decision F39), this + * installation's `lazurio browser serve` on the entry's loopback port for the + * gateway's origin. A long-running service, restarted whenever it ends. */ +export function renderBrowserViewUnit( + entry: BrowserEntry, + selector: string, +): string { return [ unitMarker, "[Unit]", - "Description=Lazurio Environment browser view: the agent-browser dashboard behind the gateway", - "ConditionFileIsExecutable=%h/.local/bin/agent-browser", + "Description=Lazurio Environment browser view: one tab of a person is one tab of the Environment browser", + `After=${browserUnit}`, "", "[Service]", - "Type=oneshot", - "RemainAfterExit=yes", - "KillMode=process", + "Type=simple", `Environment=PATH=${unitPath}`, - `Environment=AGENT_BROWSER_CDP=${browserCdpPort}`, - `ExecStart=%h/.local/bin/agent-browser dashboard start --port ${entry.listenPort} --allowed-origins ${systemdQuote(entry.origin)}`, - "ExecStop=-%h/.local/bin/agent-browser dashboard stop", - "TimeoutStartSec=60", + `ExecStart=${[ + selector, + "browser", + "serve", + "--port", + String(entry.listenPort), + "--origin", + entry.origin, + ] + .map(systemdQuote) + .join(" ")}`, + "Restart=always", + "RestartSec=2", "", "[Install]", "WantedBy=default.target", @@ -175,11 +200,12 @@ export function renderBrowserViewUnit(entry: BrowserEntry): string { export function renderBrowserUnits( entry: BrowserEntry, + selector: string, ): Readonly> { return Object.freeze({ [displayUnit]: renderDisplayUnit(), [browserUnit]: renderBrowserUnit(), - [browserViewUnit]: renderBrowserViewUnit(entry), + [browserViewUnit]: renderBrowserViewUnit(entry, selector), }); } @@ -211,15 +237,18 @@ export const environmentBrowserFailed = ( Object.freeze({ state: "failed", step, next: failedNext }); /** Write what differs, reread the manager once, enable and start all three. - * `start` of an active unit changes nothing, so the screen and the browser - * keep running across every install and update. Only the view is restarted, - * and only when its own text changed (the gateway moved its port or - * origin): a new dashboard costs a viewer one reload, nothing else. */ + * `start` of an active unit changes nothing, so the screen keeps running + * across every install and update. The view is restarted when its own text + * changed (a new installation path, port or origin): a viewer reconnects. + * The browser is restarted when its own text changed (F39 point 10): its + * windows close, agents open theirs again, the profile keeps its sign-ins. */ export async function installEnvironmentBrowser( input: Readonly<{ directory: string; hosted: boolean; entry: BrowserEntry | undefined; + /** This installation's `lazurio`, which the view unit runs. */ + selector: string; run: ProcessRunner; env: Readonly>; }>, @@ -229,9 +258,12 @@ export async function installEnvironmentBrowser( return Object.freeze({ state: "skipped-not-declared" }); const failed = environmentBrowserFailed; const command = { run: input.run, env: input.env }; - const texts = renderBrowserUnits(input.entry); + const texts = renderBrowserUnits(input.entry, input.selector); const changed: BrowserUnit[] = []; try { + const home = input.env.HOME; + if (home === undefined || !home.startsWith("/")) return failed("unit"); + await writeBrowserExtension(home); for (const unit of browserUnits) { const existing = await readFile( join(input.directory, unit), @@ -257,11 +289,12 @@ export async function installEnvironmentBrowser( return failed("reload"); if (!(await systemctl(command, "enable", ...browserUnits))) return failed("enable"); - if ( - changed.includes(browserViewUnit) && - !(await systemctl(command, "try-restart", browserViewUnit)) - ) - return failed("start"); + for (const unit of [browserUnit, browserViewUnit] as const) + if ( + changed.includes(unit) && + !(await systemctl(command, "try-restart", unit)) + ) + return failed("start"); if (!(await systemctl(command, "start", ...browserUnits))) return failed("start"); return Object.freeze({ state: "enabled" }); @@ -290,6 +323,7 @@ export async function convergeEnvironmentBrowser( directory, hosted, entry: hosted ? await input.entry().catch(() => undefined) : undefined, + selector: layout(input.base).selector, run: input.run, env: input.env, }); @@ -389,7 +423,11 @@ export async function observeEnvironmentBrowser( }; if (!(await answers(`http://127.0.0.1:${browserCdpPort}/json/version`))) return warn("browser-not-answering"); - if (!(await answers(`http://127.0.0.1:${input.entry.listenPort}/`))) + if ( + !(await answers( + `http://127.0.0.1:${input.entry.listenPort}/.lazurio/health`, + )) + ) return warn("browser-view-not-answering"); return Object.freeze({ outcome: "ok" as const }); } diff --git a/src/browser/view.ts b/src/browser/view.ts index 109a5161b..8ef89d262 100644 --- a/src/browser/view.ts +++ b/src/browser/view.ts @@ -1,27 +1,18 @@ -import { join } from "node:path"; -import type { ProcessRunner } from "../update/self-check"; -import { type BrowserEntry, browserCdpPort } from "./units"; +import type { BrowserEntry } from "./units"; -/** The person's view of the Environment browser (decision F38): the - * agent-browser dashboard on the gateway's `browser.` origin. The dashboard - * keeps its own lock: an access token that a browser must present once in the - * URL fragment, after which it is a host-only cookie of the view's origin. The - * token never leaves the Environment otherwise: the Launchpad, behind the - * same gateway admission, reads it from `agent-browser dashboard start` - * (which only reprints it while the dashboard runs with the same settings) - * and hands it over in the fragment of a redirect or a same-origin JSON - * answer, the way it pairs T3 Code and Lazurio MausBot. */ +/** Where a person sees the Environment browser (decisions F38 and F39): the + * people's view on the gateway's `browser.` origin. The address of one remote + * tab is `/t/`, and `/` opens a new one. Neither + * carries a token: the gateway's session is the only admission (F39 point + * 8). The Launchpad answers these addresses for the panels and the links. */ /** An agent-browser session name as agent-browser accepts it - * (`[A-Za-z0-9_-]`, `cli/src/validation.rs`), at most 64 characters, the - * dashboard's limit. */ + * (`[A-Za-z0-9_-]`, `cli/src/validation.rs`), at most 64 characters. */ export const isBrowserSession = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(value); -const isAccessToken = (value: string) => /^[0-9a-f]{64}$/.test(value); - /** The answer of `GET /.lazurio/browser.json`: where the view of one - * session (or of all of them) is, or that this Environment has none. */ + * thread's tab (or a new tab) is, or that this Environment has none. */ export type BrowserView = | Readonly<{ available: true; view: string; session: string | null }> | Readonly<{ available: false; reason: BrowserViewReason }>; @@ -29,131 +20,24 @@ export type BrowserView = export const browserViewReasons = [ /** The recorded entry has no `browser`: no view is routed here. */ "not-declared", - /** agent-browser is missing, refused the settings, or answered no token. */ + /** The thread's window could not be found or opened. */ "view-unavailable", ] as const; export type BrowserViewReason = (typeof browserViewReasons)[number]; export type BrowserViewSeams = Readonly<{ - run: ProcessRunner; - fetch: (url: string, init: RequestInit) => Promise; - env: Readonly>; - /** Opens the named session's window when it has none, exactly as `lazurio - * browser window` does (`ensureThreadWindow`), so a view asked for a thread - * shows that thread's window, the one its agent works in. Absent, or when - * it fails: the view of every window. */ - openWindow?: ((session: string) => Promise) | undefined; + /** The session's window, opened when it has none, exactly as `lazurio + * browser window` does (`ensureThreadWindow`): the person and the thread's + * agent always see the same tab. */ + openWindow: (session: string) => Promise>; }>; -const commandTimeoutMs = 15_000; -const sessionsTimeoutMs = 3_000; -// How long a window just opened may take to appear in the dashboard's list. -const openedSessionDeadlineMs = 3_000; -const openedSessionPollMs = 200; - -/** The dashboard's access token: `dashboard start` with exactly the unit's - * settings answers the running dashboard's URL (and starts it, the same way - * the unit would, when it is not running). */ -async function accessToken( - entry: BrowserEntry, - seams: BrowserViewSeams, -): Promise { - const home = seams.env.HOME; - if (home === undefined) return null; - const environment: Record = { - HOME: home, - PATH: [ - join(home, ".local", "bin"), - "/usr/local/bin", - "/usr/bin", - "/bin", - ].join(":"), - AGENT_BROWSER_CDP: String(browserCdpPort), - }; - if (seams.env.XDG_RUNTIME_DIR !== undefined) - environment.XDG_RUNTIME_DIR = seams.env.XDG_RUNTIME_DIR; - const answer = await seams - .run( - [ - join(home, ".local", "bin", "agent-browser"), - "dashboard", - "start", - "--port", - String(entry.listenPort), - "--allowed-origins", - entry.origin, - "--json", - ], - commandTimeoutMs, - environment, - ) - .catch(() => null); - if (answer === null || answer === "timeout" || answer.exitCode !== 0) - return null; - try { - const parsed = JSON.parse(answer.stdout) as { - data?: { access_urls?: unknown }; - }; - const urls = parsed.data?.access_urls; - if (!Array.isArray(urls)) return null; - for (const url of urls) { - if (typeof url !== "string") continue; - const prefix = `${entry.origin}/#dashboard-access-token=`; - if (!url.startsWith(prefix)) continue; - const token = url.slice(prefix.length); - if (isAccessToken(token)) return token; - } - } catch {} - return null; -} - -/** The stream port of one session, from the dashboard's own list on - * loopback (where its rule needs no token, only an Origin of the same - * loopback authority). Null when the session is not running yet. */ -async function sessionPort( - entry: BrowserEntry, - session: string, - seams: BrowserViewSeams, -): Promise { - const loopback = `http://127.0.0.1:${entry.listenPort}`; - try { - const response = await seams.fetch(`${loopback}/api/sessions`, { - headers: { Origin: loopback }, - redirect: "error", - signal: AbortSignal.timeout(sessionsTimeoutMs), - }); - if (!response.ok) return null; - const list: unknown = await response.json(); - if (!Array.isArray(list)) return null; - for (const item of list) { - if (typeof item !== "object" || item === null) continue; - const { session: name, port } = item as Record; - if ( - name === session && - typeof port === "number" && - Number.isInteger(port) && - port >= 1 && - port <= 65535 - ) - return port; - } - } catch {} - return null; -} - -/** The view's URL: the dashboard, the session's window selected by its - * stream port when it runs, and the access token in the fragment. The - * `view=.html` parameter works around agent-browser serving `/?port=…` as - * `application/octet-stream` (vercel-labs/agent-browser PR #2046): the - * dashboard derives the type from the request target's ending; the - * dashboard's page ignores the parameter. */ +/** The view of one remote tab, or of a new one without a target. */ export function browserViewUrl( origin: string, - token: string, - port: number | null, + targetId: string | null, ): string { - const query = port === null ? "" : `?port=${port}&view=.html`; - return `${origin}/${query}#dashboard-access-token=${token}`; + return targetId === null ? `${origin}/` : `${origin}/t/${targetId}`; } export async function resolveBrowserView( @@ -163,30 +47,20 @@ export async function resolveBrowserView( ): Promise { if (entry === undefined) return Object.freeze({ available: false, reason: "not-declared" }); - const token = await accessToken(entry, seams); - if (token === null) + // Without a session (the Launchpad's panel): a new remote tab. The view of + // every window is retired (root decision 0191 point 18). + if (session === null) + return Object.freeze({ + available: true, + view: browserViewUrl(entry.origin, null), + session: null, + }); + const window = await seams.openWindow(session).catch(() => null); + if (window === null) return Object.freeze({ available: false, reason: "view-unavailable" }); - let port = session === null ? null : await sessionPort(entry, session, seams); - // A thread asks for its own window before its agent has opened one (the - // Browser panel of a new T3 Code thread): open it now, so the person and - // the agent look at the same window instead of the list of every window. - if (session !== null && port === null && seams.openWindow !== undefined) { - const opened = await seams.openWindow(session).then( - () => true, - () => false, - ); - const deadline = performance.now() + openedSessionDeadlineMs; - while (opened && port === null && performance.now() < deadline) { - port = await sessionPort(entry, session, seams); - if (port === null) - await new Promise((resolve) => - setTimeout(resolve, openedSessionPollMs), - ); - } - } return Object.freeze({ available: true, - view: browserViewUrl(entry.origin, token, port), + view: browserViewUrl(entry.origin, window.targetId), session, }); } diff --git a/src/browser/window.ts b/src/browser/window.ts index 16e73a4b8..d857ac5c9 100644 --- a/src/browser/window.ts +++ b/src/browser/window.ts @@ -222,15 +222,16 @@ async function ensureThreadWindowLocked( "tab", targetId, ]).catch(() => "timeout" as const); + // Pinned with a command that changes nothing. The page keeps the window's + // size, which the people's view sets to the person's page area (F39 point + // 3); a fixed viewport served only the dashboard's pointer mapping. const pinned = ok(bound) ? await agentBrowser(seams, [ "--session", session, "--pin-tab", - "set", - "viewport", - String(browserWindowSize.width), - String(browserWindowSize.height), + "get", + "url", ]).catch(() => "timeout" as const) : "timeout"; if (!ok(bound) || !ok(pinned)) throw new BrowserWindowFailure("bind-failed"); diff --git a/src/cli.ts b/src/cli.ts index 86d93fe32..49f39a93e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,5 +1,6 @@ import { parseArgs } from "node:util"; import { browserHelp, runBrowserCommand } from "./browser/cli"; +import { runViewService } from "./browser/people/serve"; import { ContentUsageError, contentHelp, @@ -194,7 +195,10 @@ export async function runCli(args: string[]): Promise { const code = await runOtherCommand(args); // Every other command ends with the one-line notice (docs/update.md // "Surfaces"): from `last-check.json` only, on stderr, never the network. - if (args[0] !== "launchpad") { + if ( + args[0] !== "launchpad" && + !(args[0] === "browser" && args[1] === "serve") + ) { const notice = await noticeAfterCommand(); if (notice) console.error(notice); } @@ -205,6 +209,10 @@ async function runOtherCommand(args: string[]): Promise { // The link alone on stdout; the notice follows on stderr. if (args[0] === "chat") return emit(await runChatCommand(args.slice(1), processContext())); + // The Environment browser's people's view (decision F39): the service the + // view unit runs until it is stopped. + if (args[0] === "browser" && args[1] === "serve") + return runViewService(args.slice(2), process.env); // The Environment browser (decision F38): a thread's window and the link to // its view. if (args[0] === "browser") diff --git a/src/folder/manual.ts b/src/folder/manual.ts index 33ff896aa..a5717e80d 100644 --- a/src/folder/manual.ts +++ b/src/folder/manual.ts @@ -1888,39 +1888,39 @@ function operatorClient( ]; } -// The Environment browser (root decision 0191, decision F38): one shared -// Chromium per Environment, a window per thread through `lazurio browser -// window`, worked in with agent-browser, watched and taken over by the person -// through the view the gateway routes. Rendered only where the recorded entry -// has the view. +// The Environment browser (root decision 0191, decisions F38 and F39): one +// shared Chromium per Environment, a window per thread through `lazurio +// browser window`, worked in with agent-browser, and the people's view the +// gateway routes, where the Operator works in the same tab at the same time. +// Rendered only where the recorded entry has the view. function environmentBrowser(preset: PresetName): readonly Text[] { const team = preset === "hosted-organization-team"; return [ t("## Prohlížeč Environmentu", "## The Environment browser"), t( - `Tenhle Environment má jeden sdílený prohlížeč: Chromium na virtuální obrazovce \`:1\` s jedním trvalým profilem (decision 0191). Přihlášení v něm patří Environmentu: sdílí je všechna vlákna, agenti a boti${team ? " a celý Team" : ""}. Pro práci v prohlížeči používej tenhle prohlížeč, ne vlastní: Operátor tě v něm vidí a může převzít ovládání.`, - `This Environment has one shared browser: Chromium on the virtual screen \`:1\` with one persistent profile (decision 0191). Its sign-ins belong to the Environment: every thread, agent and bot${team ? " and the whole Team" : ""} share them. Use this browser for browser work, not one of your own: the Operator sees you in it and can take over.`, + `Tenhle Environment má jeden sdílený prohlížeč: Chromium na virtuální obrazovce \`:1\` s jedním trvalým profilem (decision 0191). Přihlášení v něm patří Environmentu: sdílí je všechna vlákna, agenti a boti${team ? " a celý Team" : ""}. Pro práci v prohlížeči používej tenhle prohlížeč, ne vlastní: Operátor tě v něm vidí a pracuje s tebou ve stejné záložce.`, + `This Environment has one shared browser: Chromium on the virtual screen \`:1\` with one persistent profile (decision 0191). Its sign-ins belong to the Environment: every thread, agent and bot${team ? " and the whole Team" : ""} share them. Use this browser for browser work, not one of your own: the Operator sees you in it and works with you in the same tab.`, ), blank, t( - "- **„Otevřený prohlížeč“ je tenhle.** Když Operátor mluví o prohlížeči, který má otevřený, nebo se ptá, co v něm je, myslí prohlížeč Environmentu: ten vidí v panelu Browser webového T3 Code, v pravém panelu Launchpadu i z tvého odkazu. Panel Browser ve vlákně T3 ukazuje okno tohoto vlákna. Nástroje náhledu T3 Code (`preview_*`) a vestavěný prohlížeč desktopových aplikací ovládají prohlížeč na Operátorově počítači, ne tenhle; ve webovém T3 nejsou. Co je v oknech Environmentu otevřené, vypíše `curl -s http://127.0.0.1:9222/json/list` (adresa a titulek každé stránky).", - "- **\"The open browser\" is this one.** When the Operator talks about the browser they have open, or asks what is in it, they mean the Environment browser: they see it in web T3 Code's Browser panel, in the Launchpad's right panel and from your link. The Browser panel of a T3 thread shows that thread's window. T3 Code's preview tools (`preview_*`) and the built-in browser of desktop apps drive a browser on the Operator's computer, not this one; web T3 has none. `curl -s http://127.0.0.1:9222/json/list` lists what the Environment's windows have open (each page's address and title).", + "- **„Otevřený prohlížeč“ je tenhle.** Když Operátor mluví o prohlížeči, který má otevřený, nebo se ptá, co v něm je, myslí prohlížeč Environmentu: ten vidí v panelu Browser webového T3 Code, v pravém panelu Launchpadu i z tvého odkazu. Panel Browser ve vlákně T3 ukazuje záložku tohoto vlákna. Nástroje náhledu T3 Code (`preview_*`) a vestavěný prohlížeč desktopových aplikací ovládají prohlížeč na Operátorově počítači, ne tenhle; ve webovém T3 nejsou. Co je v oknech Environmentu otevřené, vypíše `curl -s http://127.0.0.1:9222/json/list` (adresa a titulek každé stránky).", + "- **\"The open browser\" is this one.** When the Operator talks about the browser they have open, or asks what is in it, they mean the Environment browser: they see it in web T3 Code's Browser panel, in the Launchpad's right panel and from your link. The Browser panel of a T3 thread shows that thread's tab. T3 Code's preview tools (`preview_*`) and the built-in browser of desktop apps drive a browser on the Operator's computer, not this one; web T3 has none. `curl -s http://127.0.0.1:9222/json/list` lists what the Environment's windows have open (each page's address and title).", ), t( - "- **Tvoje okno.** Na začátku práce s prohlížečem spusť `lazurio browser window` (případně `--url `). Otevře okno tvého vlákna, nebo najde to, které už máš, a vypíše jméno sezení, odkaz na pohled a příkaz, kterým v okně pracuješ. Jméno sezení plyne z vlákna (`AGENT_BROWSER_SESSION`, `CODEX_THREAD_ID`, `CLAUDE_CODE_SESSION_ID`); jinak ho zadej `--session `.", - "- **Your window.** When you start browser work, run `lazurio browser window` (optionally `--url
`). It opens your thread's window, or finds the one you have, and prints the session name, the link to the view and the command to work in the window with. The session name follows from the thread (`AGENT_BROWSER_SESSION`, `CODEX_THREAD_ID`, `CLAUDE_CODE_SESSION_ID`); otherwise give it with `--session `.", + "- **Tvoje okno.** Na začátku práce s prohlížečem spusť `lazurio browser window` (případně `--url `). Otevře okno tvého vlákna, nebo najde to, které už máš, a vypíše jméno sezení, odkaz na tvou záložku v pohledu (`https://browser.…/t/`) a příkaz, kterým v okně pracuješ. Jméno sezení plyne z vlákna (`AGENT_BROWSER_SESSION`, `CODEX_THREAD_ID`, `CLAUDE_CODE_SESSION_ID`); jinak ho zadej `--session `.", + "- **Your window.** When you start browser work, run `lazurio browser window` (optionally `--url
`). It opens your thread's window, or finds the one you have, and prints the session name, the link to your tab in the people's view (`https://browser.…/t/`) and the command to work in the window with. The session name follows from the thread (`AGENT_BROWSER_SESSION`, `CODEX_THREAD_ID`, `CLAUDE_CODE_SESSION_ID`); otherwise give it with `--session `.", ), t( - '- **Práce v okně.** `agent-browser --cdp 9222 --session `: například `open `, `snapshot -i`, `click @e1`, `fill @e2 "text"`, `screenshot `, `get url`. Vždy s `--cdp 9222` a svým `--session`: bez `--cdp` by agent-browser spustil vlastní prohlížeč bez přihlášení Environmentu a Operátor by ho neviděl. Nepoužívej `window new` (otevře izolované okno bez přihlášení), `--profile`, `--headed` ani `set device`.', - '- **Working in the window.** `agent-browser --cdp 9222 --session `: for example `open `, `snapshot -i`, `click @e1`, `fill @e2 "text"`, `screenshot `, `get url`. Always with `--cdp 9222` and your `--session`: without `--cdp` agent-browser starts a browser of its own, without the Environment\'s sign-ins and invisible to the Operator. Do not use `window new` (it opens an isolated window without sign-ins), `--profile`, `--headed` or `set device`.', + '- **Práce v okně.** `agent-browser --cdp 9222 --session `: například `open `, `snapshot -i`, `click @e1`, `fill @e2 "text"`, `screenshot `, `get url`. Vždy s `--cdp 9222` a svým `--session`: bez `--cdp` by agent-browser spustil vlastní prohlížeč bez přihlášení Environmentu a Operátor by ho neviděl. Nepoužívej `window new` (otevře izolované okno bez přihlášení), `--profile`, `--headed`, `set device` ani `set viewport`: velikost stránky určuje pohled Operátora.', + "- **Working in the window.** `agent-browser --cdp 9222 --session `: for example `open `, `snapshot -i`, `click @e1`, `fill @e2 \"text\"`, `screenshot `, `get url`. Always with `--cdp 9222` and your `--session`: without `--cdp` agent-browser starts a browser of its own, without the Environment's sign-ins and invisible to the Operator. Do not use `window new` (it opens an isolated window without sign-ins), `--profile`, `--headed`, `set device` or `set viewport`: the Operator's view sets the page size.", ), t( - "- **Pohled pro Operátora.** Když začneš, pošli Operátorovi odkaz, který vypíše `lazurio browser window` (nebo `lazurio browser link`). Otevře ho v pravém panelu ChatGPT Desktopu (dřív Codex Desktop), v panelu Browser webového T3 Code, v pravém panelu Launchpadu, v tabu nebo na telefonu připojeném do tailnetu a uvidí tvoje okno živě.", - "- **The Operator's view.** When you start, send the Operator the link `lazurio browser window` (or `lazurio browser link`) prints. They open it in ChatGPT Desktop's right panel (formerly Codex Desktop), in web T3 Code's Browser panel, in the Launchpad's right panel, in a tab or on a phone on the tailnet, and see your window live.", + "- **Pohled pro Operátora.** Když začneš, pošli Operátorovi odkaz na svou záložku, který vypíše `lazurio browser window`. Otevře ho v pravém panelu ChatGPT Desktopu (dřív Codex Desktop), v panelu Browser webového T3 Code, v pravém panelu Launchpadu, v tabu nebo na telefonu připojeném do tailnetu: jedna jeho záložka je přesně tvoje záložka a pracujete v ní současně, ovládání se nepředává. Holou adresu `https://browser.…/` neposílej, ta vždy otevře novou prázdnou záložku. Okno, které stránka otevře jako pop-up (třeba přihlášení přes Google), Operátor uvidí ve stejné záložce; novou záložku, kterou stránka otevře, mu pohled nabídne.", + "- **The Operator's view.** When you start, send the Operator the link to your tab that `lazurio browser window` prints. They open it in ChatGPT Desktop's right panel (formerly Codex Desktop), in web T3 Code's Browser panel, in the Launchpad's right panel, in a tab or on a phone on the tailnet: one tab of theirs is exactly your tab, and you both work in it at the same time; control is never handed over. Never send the bare address `https://browser.…/`, which always opens a new blank tab. A window the page opens as a pop-up (Sign in with Google, say) shows in the same tab of the Operator; a new tab the page opens is offered to them by the view.", ), t( - "- **Přihlášení a ověření.** Na přihlašovací stránce, 2FA nebo CAPTCHA se zastav, řekni Operátorovi, ať v pohledu převezme ovládání, a počkej na jeho odpověď. Hesla ani kódy nevyplňuj sám a nežádej o ně v chatu; dokud ovládá Operátor, okno neovládej a nefoť. Přihlášení pak platí pro celý Environment: neodhlašuj se a nemaž cookies ani data prohlížeče.", - "- **Sign-in and verification.** At a sign-in page, 2FA or a CAPTCHA, stop, ask the Operator to take over in the view and wait for their answer. Never fill in passwords or codes yourself or ask for them in the chat; while the Operator is in control, do not drive or capture the window. The sign-in then holds for the whole Environment: do not sign out or clear cookies or browser data.", + "- **Přihlášení a ověření.** Na přihlašovací stránce, 2FA nebo CAPTCHA se zastav, pošli Operátorovi odkaz na svou záložku, požádej ho, ať přihlášení v ní dokončí, a počkej na jeho odpověď. Hesla ani kódy nevyplňuj sám a nežádej o ně v chatu. Dokud se Operátor přihlašuje, nepiš do stejných polí a záložku nefoť. Přihlášení pak platí pro celý Environment: neodhlašuj se a nemaž cookies ani data prohlížeče. Passkey zatím prohlížeč odmítá, dokud nebude mít Environment svůj Bitwarden; zvol jinou cestu přihlášení.", + "- **Sign-in and verification.** At a sign-in page, 2FA or a CAPTCHA, stop, send the Operator the link to your tab, ask them to finish the sign-in there and wait for their answer. Never fill in passwords or codes yourself or ask for them in the chat. While the Operator signs in, do not type into the same fields and do not capture the tab. The sign-in then holds for the whole Environment: do not sign out or clear cookies or browser data. Until the Environment has its Bitwarden, the browser declines passkeys; choose another way to sign in.", ), t( "- **`localhost`.** Prohlížeč Environmentu otevře i `localhost` tohoto Environmentu, takže rozpracovanou stránku Operátor uvidí v pohledu.", diff --git a/src/folder/render.ts b/src/folder/render.ts index 482752598..2a78ca833 100644 --- a/src/folder/render.ts +++ b/src/folder/render.ts @@ -27,7 +27,7 @@ import { ownDataValue, stateFields } from "./state-fields"; // future persisted preference schemas. Every change of the generated bytes // moves it to the next number; tests/folder-template-revision.test.ts pins the // rendered digest of the current one (decision F14, addendum 2026-10-03). -export const instructionTemplateRevision = "base-instructions-30"; +export const instructionTemplateRevision = "base-instructions-31"; // Template revisions are ordered by their number. A Folder rendered by an // older revision is re-rendered by the next change of the generated Folder diff --git a/src/launchpad/browser-panel-view.ts b/src/launchpad/browser-panel-view.ts index a1ecf372b..ef94226f3 100644 --- a/src/launchpad/browser-panel-view.ts +++ b/src/launchpad/browser-panel-view.ts @@ -2,17 +2,17 @@ import type { PublicEntry } from "./chat"; import type { MessageKey } from "./messages"; // The Launchpad's right panel (decision F38's addendum of 2026-10-05, root -// decision 0191 point 8a, #207): the view of every window of the Environment -// browser, the same view every app of the Environment embeds in its right -// panel; the Launchpad builds no viewer of its own. The panel is offered -// only where the recorded entry routes the view (`browserOrigin`): never on a -// workstation, which has no entry, and never in Recovery mode. Opening it -// asks this origin for `GET /.lazurio/browser.json` without a session (every -// window) and embeds the view only when its address is an https URL on -// exactly the recorded origin. The address carries the dashboard's access -// token in its fragment, so it is never stored: it lives in the frame and in -// the panel's link while the panel is open, and it is asked for again on -// every opening and every reload. Pure but for the fetch passed in. +// decision 0191 points 8a and 18, #207): the people's view of the Environment +// browser (decision F39), the same view every app of the Environment embeds +// in its right panel; the Launchpad builds no viewer of its own. The panel is +// offered only where the recorded entry routes the view (`browserOrigin`): +// never on a workstation, which has no entry, and never in Recovery mode. +// Opening it asks this origin for `GET /.lazurio/browser.json` without a +// session, which answers the view's `/`: a new remote tab, closed again a +// short while after the panel closes. The view of every window is retired. +// The panel embeds the view only when its address is an https URL on exactly +// the recorded origin, and asks again on every opening and every reload. +// Pure but for the fetch passed in. type Copy = Readonly>; type Fetch = (input: string, init: RequestInit) => Promise; @@ -23,8 +23,8 @@ export const browserViewDocumentPath = "/.lazurio/browser.json"; * browser to the view, through the gateway's sign-in when the session has * expired. A frame cannot show that sign-in page; a new tab can. */ export const browserHandOverPath = "/.lazurio/browser"; -/** How long the panel waits for the answer: the server may first start - * agent-browser's dashboard, which it gives 15 s. */ +/** How long the panel waits for the answer; without a session it is + * immediate. */ export const browserViewReadMs = 20_000; /** The frame's attributes. `allow` lets the view use the clipboard (copying @@ -79,8 +79,8 @@ export function browserViewAddress( } /** What the open panel shows: the answer still coming, the view, an - * Environment browser that answered it is not available (agent-browser - * missing, or no token), or no answer that can be used (an expired session, + * Environment browser that answered it is not available, or no answer that + * can be used (an expired session, * a refusal, a timeout, an answer of another shape). */ export type BrowserPanelState = | Readonly<{ kind: "loading" }> diff --git a/src/launchpad/server.ts b/src/launchpad/server.ts index b164c455b..40c7c6925 100644 --- a/src/launchpad/server.ts +++ b/src/launchpad/server.ts @@ -273,9 +273,6 @@ export async function startLaunchpad( // account's agent-browser and the dashboard on loopback. Trusted // composition, never HTTP input; tests supply stubs. browserViewSeams: BrowserViewSeams = Object.freeze({ - run: runProcess, - fetch: (url: string, init: RequestInit) => fetch(url, init), - env: process.env, openWindow: (session: string) => ensureThreadWindow(session, undefined, cdpSeams(process.env, runProcess)), }), @@ -550,12 +547,12 @@ export async function startLaunchpad( url.pathname === browserViewDocumentPath || url.pathname === browserViewPath ) { - // The Environment browser's view (decision F38), behind the same - // admission as every read: the dashboard's address with the window - // of one agent-browser session selected and the dashboard's access - // token in the fragment. The JSON answer is for the panels; the - // hand-over redirects a link's browser there. Only `session`, and - // only a name agent-browser accepts. + // The Environment browser's people's view (decisions F38 and F39), + // behind the same admission as every read: the address of one + // agent-browser session's tab, opened when it has none, or of a new + // remote tab without a session. The JSON answer is for the panels; + // the hand-over redirects a link's browser there. Only `session`, + // and only a name agent-browser accepts. if (request.method !== "GET") return response({ error: "method-not-allowed" }, 405); if (closing) return response({ error: "closing" }, 503); diff --git a/tests/__snapshots__/folder-manual.test.ts.snap b/tests/__snapshots__/folder-manual.test.ts.snap index 6e9930526..b9db9cfef 100644 --- a/tests/__snapshots__/folder-manual.test.ts.snap +++ b/tests/__snapshots__/folder-manual.test.ts.snap @@ -2,7 +2,7 @@ exports[`a personal VM names reachable peers neutrally from the record and requires the Operator's confirmation 1`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`example\` (personal-vm). - Owner: the person with GitHub login \`example\` (id 12345). @@ -85,7 +85,7 @@ Peers this Environment may reach over SSH, exactly as the handover records them: exports[`a personal VM names reachable peers neutrally from the record and requires the Operator's confirmation 2`] = ` "# Tenhle Environment - + ## Identita podle zaznamenaného handoveru - Název: \`example\` (personal-vm). - Owner: člověk s GitHub loginem \`example\` (id 12345). @@ -168,7 +168,7 @@ Peers, na které odsud smíš přes SSH, přesně jak je zaznamenal handover: exports[`rendered manual/this-machine.md snapshot: local 1`] = ` "# This Environment - + ## Identity, from the recorded handover - The Operator's own workstation. No handover was recorded; the signed-in user is both Owner and Operator. @@ -205,7 +205,7 @@ MCP servers come after the catalog CLIs: discover in your harness which ones it exports[`rendered manual/this-machine.md snapshot: hosted-personal 1`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`example\` (personal-vm). - Owner: the person with GitHub login \`example\` (id 12345). @@ -280,7 +280,7 @@ The handover records no peer this Environment may reach over SSH; do not look fo exports[`rendered manual/this-machine.md snapshot: hosted-organization-personal 1`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`workspace\` (workspace-vm). - Owner: Organization \`example\`. @@ -350,7 +350,7 @@ MCP servers come after the catalog CLIs: discover in your harness which ones it exports[`rendered manual/this-machine.md snapshot: hosted-organization-team 1`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`workspace\` (workspace-vm). - Owner: Organization \`example\`, Team \`sample-team\`. @@ -422,7 +422,7 @@ MCP servers come after the catalog CLIs: discover in your harness which ones it exports[`rendered manual/this-machine.md snapshot: hosted-organization-steward 1`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`workspace\` (workspace-vm). - Owner: Organization \`example\`. @@ -510,7 +510,7 @@ MCP servers come after the catalog CLIs: discover in your harness which ones it exports[`rendered manual/lazurio.md snapshot 1`] = ` "# Lazurio - + ## What Lazurio is Lazurio is the shared framework in which people and machines work together on the work of companies (Organizations, Teams and modules) and on a person's own affairs (Personalspace and the optional Buddy). Lazurio is the product's only current name; earlier names remain only in historical records (decision 0128). @@ -569,7 +569,7 @@ These layers are not copied into each other automatically; a chat is a cache. exports[`rendered manual/lazurio.md snapshot 2`] = ` "# Lazurio - + ## Co je Lazurio Lazurio je sdílený rámec, ve kterém lidé a stroje společně pracují na práci firem (Organizace, Teamy a moduly) i na osobních záležitostech člověka (Personalspace a volitelný Buddy). Lazurio je jediný aktuální název produktu; dřívější názvy zůstávají jen v historických záznamech (decision 0128). @@ -628,7 +628,7 @@ Tyhle vrstvy se do sebe automaticky nekopírují; chat je jen cache. exports[`rendered manual/working-here.md snapshot 1`] = ` "# Working here - + ## Draft, Publication, Release - **Draft** is your work: a revertible and, above all, editable piece of work, a change in a worktree, a drafted e-mail, an open pull request. - **Publication** is the act that makes a Draft hard to revert or visible outside: a merge, sending, deploying; in data applications already the commit and push of the “Publish changes” button. Publication belongs to the Operator: you perform it only on their explicit instruction, which holds in the current thread and does not carry over. Whether the Operator may approve it is decided by their live rights to the exact operation. @@ -775,7 +775,7 @@ Do the work so that they have it: understandable, reversible, with room to adjus exports[`rendered manual/working-here.md snapshot 2`] = ` "# Jak se tu pracuje - + ## Draft, Publikace, Release - **Draft** je tvoje práce: vratný a hlavně editovatelný kus práce, změna ve worktree, rozepsaný e-mail, otevřený pull request. - **Publikace** je akt, kterým se Draft stává těžko vratným nebo viditelným navenek: merge, odeslání, nasazení; v datových aplikacích už commit a push tlačítkem „Publikovat změny“. Publikace patří Operátorovi: provedeš ji jen na jeho explicitní pokyn, který platí v aktuálním threadu a nepřenáší se dál. Jestli ji Operátor smí schválit, rozhodují jeho živá práva k přesné operaci. @@ -922,7 +922,7 @@ Odveď práci tak, aby ho měl: srozumitelně, vratně, s prostorem k úpravě. exports[`rendered manual/roles.md snapshot 1`] = ` "# Roles - + Names authorize nothing. In every role below, what a person may do is decided by their live rights at the providers, above all on GitHub (decisions 0102/0103). ## Operator (Operátor) @@ -958,7 +958,7 @@ The term “AI Kolega”, an AI with its own seat and authority like a person, i exports[`rendered manual/roles.md snapshot 2`] = ` "# Role - + Názvy nic neautorizují. U každé role níže rozhodují o tom, co člověk smí, jeho živá práva u providerů, především na GitHubu (decisions 0102/0103). ## Operátor @@ -994,7 +994,7 @@ Pojem „AI Kolega“, tedy AI s vlastním seatem a pravomocemi jako člověk, j exports[`rendered manual/glossary.md snapshot 1`] = ` "# Glossary - + English terms first; the Czech term the product and its people use is in parentheses. | Term | Meaning | @@ -1048,7 +1048,7 @@ English terms first; the Czech term the product and its people use is in parenth exports[`rendered manual/glossary.md snapshot 2`] = ` "# Slovník - + Nejdřív český pojem, v závorce anglický, který používají produkt a technické identifikátory. | Pojem | Význam | @@ -1102,7 +1102,7 @@ Nejdřív český pojem, v závorce anglický, který používají produkt a tec exports[`rendered manual/troubleshooting.md snapshot 1`] = ` "# Installation, updates and troubleshooting - + ## Lazurio in this Environment Lazurio is installed here: the product in \`~/.local/share/lazurio/\` with the command \`lazurio\`, and this Folder with its generated files \`AGENTS.md\`, \`manual/\` and \`.lazurio/\`. It is installed only by its installer and updated with \`lazurio update\`. @@ -1179,7 +1179,7 @@ An open technical problem belongs in a GitHub Issue of the exact owning reposito exports[`rendered manual/troubleshooting.md snapshot 2`] = ` "# Instalace, aktualizace a řešení problémů - + ## Lazurio na tomhle Environmentu Lazurio je tu nainstalované: produkt v \`~/.local/share/lazurio/\` s příkazem \`lazurio\` a tenhle Folder s generovanými soubory \`AGENTS.md\`, \`manual/\` a \`.lazurio/\`. Instaluje se jen svým instalátorem a aktualizuje příkazem \`lazurio update\`. @@ -1256,7 +1256,7 @@ Otevřený technický problém patří do GitHub Issue v přesném owning repozi exports[`rendered manual/troubleshooting.md snapshot: hosted-personal / cs 1`] = ` "# Instalace, aktualizace a řešení problémů - + ## Lazurio na tomhle Environmentu Lazurio je tu nainstalované a spravované: produkt v \`~/.local/share/lazurio/\` s příkazem \`~/.local/bin/lazurio\` a tenhle Folder s generovanými soubory \`AGENTS.md\`, \`manual/\` a \`.lazurio/\`. Když tě Operátor požádá „nainstaluj Lazurio“, ověř stav (\`lazurio doctor\`) a doplň jen to, co chybí, postupy níže. @@ -1326,7 +1326,7 @@ Otevřený technický problém patří do GitHub Issue v přesném owning repozi exports[`rendered manual/troubleshooting.md snapshot: hosted-personal / en 1`] = ` "# Installation, updates and troubleshooting - + ## Lazurio in this Environment Lazurio is installed and managed here: the product in \`~/.local/share/lazurio/\` with the command \`~/.local/bin/lazurio\`, and this Folder with its generated files \`AGENTS.md\`, \`manual/\` and \`.lazurio/\`. When the Operator asks you to “install Lazurio”, check the state (\`lazurio doctor\`) and complete only what is missing, with the procedures below. @@ -1396,7 +1396,7 @@ An open technical problem belongs in a GitHub Issue of the exact owning reposito exports[`rendered manual/troubleshooting.md snapshot: hosted-organization-personal / cs 1`] = ` "# Instalace, aktualizace a řešení problémů - + ## Lazurio na tomhle Environmentu Lazurio je tu nainstalované a spravované: produkt v \`~/.local/share/lazurio/\` s příkazem \`~/.local/bin/lazurio\` a tenhle Folder s generovanými soubory \`AGENTS.md\`, \`manual/\` a \`.lazurio/\`. Když tě Operátor požádá „nainstaluj Lazurio“, ověř stav (\`lazurio doctor\`) a doplň jen to, co chybí, postupy níže. @@ -1482,7 +1482,7 @@ Otevřený technický problém patří do GitHub Issue v přesném owning repozi exports[`rendered manual/troubleshooting.md snapshot: hosted-organization-personal / en 1`] = ` "# Installation, updates and troubleshooting - + ## Lazurio in this Environment Lazurio is installed and managed here: the product in \`~/.local/share/lazurio/\` with the command \`~/.local/bin/lazurio\`, and this Folder with its generated files \`AGENTS.md\`, \`manual/\` and \`.lazurio/\`. When the Operator asks you to “install Lazurio”, check the state (\`lazurio doctor\`) and complete only what is missing, with the procedures below. @@ -1568,7 +1568,7 @@ An open technical problem belongs in a GitHub Issue of the exact owning reposito exports[`rendered manual/troubleshooting.md snapshot: hosted-organization-team / cs 1`] = ` "# Instalace, aktualizace a řešení problémů - + ## Lazurio na tomhle Environmentu Lazurio je tu nainstalované a spravované: produkt v \`~/.local/share/lazurio/\` s příkazem \`~/.local/bin/lazurio\` a tenhle Folder s generovanými soubory \`AGENTS.md\`, \`manual/\` a \`.lazurio/\`. Když tě Operátor požádá „nainstaluj Lazurio“, ověř stav (\`lazurio doctor\`) a doplň jen to, co chybí, postupy níže. @@ -1655,7 +1655,7 @@ Otevřený technický problém patří do GitHub Issue v přesném owning repozi exports[`rendered manual/troubleshooting.md snapshot: hosted-organization-team / en 1`] = ` "# Installation, updates and troubleshooting - + ## Lazurio in this Environment Lazurio is installed and managed here: the product in \`~/.local/share/lazurio/\` with the command \`~/.local/bin/lazurio\`, and this Folder with its generated files \`AGENTS.md\`, \`manual/\` and \`.lazurio/\`. When the Operator asks you to “install Lazurio”, check the state (\`lazurio doctor\`) and complete only what is missing, with the procedures below. @@ -1742,7 +1742,7 @@ An open technical problem belongs in a GitHub Issue of the exact owning reposito exports[`rendered manual/troubleshooting.md snapshot: hosted-organization-steward / cs 1`] = ` "# Instalace, aktualizace a řešení problémů - + ## Lazurio na tomhle Environmentu Lazurio je tu nainstalované a spravované: produkt v \`~/.local/share/lazurio/\` s příkazem \`~/.local/bin/lazurio\` a tenhle Folder s generovanými soubory \`AGENTS.md\`, \`manual/\` a \`.lazurio/\`. Když tě Operátor požádá „nainstaluj Lazurio“, ověř stav (\`lazurio doctor\`) a doplň jen to, co chybí, postupy níže. @@ -1828,7 +1828,7 @@ Otevřený technický problém patří do GitHub Issue v přesném owning repozi exports[`rendered manual/troubleshooting.md snapshot: hosted-organization-steward / en 1`] = ` "# Installation, updates and troubleshooting - + ## Lazurio in this Environment Lazurio is installed and managed here: the product in \`~/.local/share/lazurio/\` with the command \`~/.local/bin/lazurio\`, and this Folder with its generated files \`AGENTS.md\`, \`manual/\` and \`.lazurio/\`. When the Operator asks you to “install Lazurio”, check the state (\`lazurio doctor\`) and complete only what is missing, with the procedures below. @@ -1914,7 +1914,7 @@ An open technical problem belongs in a GitHub Issue of the exact owning reposito exports[`this-machine.md renders relationships only when the binding carries them 1`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`workspace\` (workspace-vm). - Owner: Organization \`example\`. @@ -1991,7 +1991,7 @@ Lazurio enforces none of this: the Headscale policy of the Conglomerate Host doe exports[`this-machine.md renders the assignment exactly as the handover carries it 1`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`workspace\` (workspace-vm). - Owner: Organization \`example\`. @@ -2062,7 +2062,7 @@ MCP servers come after the catalog CLIs: discover in your harness which ones it exports[`this-machine.md renders the assignment exactly as the handover carries it 2`] = ` "# This Environment - + ## Identity, from the recorded handover - Name: \`workspace\` (workspace-vm). - Owner: Organization \`example\`, Team \`sample-team\`. diff --git a/tests/__snapshots__/folder-render.test.ts.snap b/tests/__snapshots__/folder-render.test.ts.snap index 1637eb2e6..e5e796235 100644 --- a/tests/__snapshots__/folder-render.test.ts.snap +++ b/tests/__snapshots__/folder-render.test.ts.snap @@ -2,7 +2,7 @@ exports[`rendered AGENTS.md snapshot: local / cs 1`] = ` "# Lazurio - + ## Tenhle Environment - Preset: \`local\` (verze 1). - Pracovní stanice Operátora bez handoveru; Owner i Operátor je přihlášený uživatel. @@ -45,7 +45,7 @@ MCP servery přicházejí na řadu až po CLI z katalogu: zjisti ve svém harnes exports[`rendered AGENTS.md snapshot: local / en 1`] = ` "# Lazurio - + ## This Environment - Preset: \`local\` (version 1). - The Operator's own workstation, no handover; the signed-in user is both Owner and Operator. @@ -88,7 +88,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`rendered AGENTS.md snapshot: hosted-personal / cs 1`] = ` "# Lazurio - + ## Tenhle Environment - Preset: \`hosted-personal\` (verze 1). - Název: \`example\` (personal-vm). @@ -137,7 +137,7 @@ MCP servery přicházejí na řadu až po CLI z katalogu: zjisti ve svém harnes exports[`rendered AGENTS.md snapshot: hosted-personal / en 1`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-personal\` (version 1). - Name: \`example\` (personal-vm). @@ -186,7 +186,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`rendered AGENTS.md snapshot: hosted-organization-personal / cs 1`] = ` "# Lazurio - + ## Tenhle Environment - Preset: \`hosted-organization-personal\` (verze 1). - Název: \`workspace\` (workspace-vm). @@ -235,7 +235,7 @@ MCP servery přicházejí na řadu až po CLI z katalogu: zjisti ve svém harnes exports[`rendered AGENTS.md snapshot: hosted-organization-personal / en 1`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-organization-personal\` (version 1). - Name: \`workspace\` (workspace-vm). @@ -284,7 +284,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`rendered AGENTS.md snapshot: hosted-organization-team / cs 1`] = ` "# Lazurio - + ## Tenhle Environment - Preset: \`hosted-organization-team\` (verze 1). - Název: \`workspace\` (workspace-vm). @@ -334,7 +334,7 @@ MCP servery přicházejí na řadu až po CLI z katalogu: zjisti ve svém harnes exports[`rendered AGENTS.md snapshot: hosted-organization-team / en 1`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-organization-team\` (version 1). - Name: \`workspace\` (workspace-vm). @@ -384,7 +384,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`rendered AGENTS.md snapshot: hosted-organization-steward / cs 1`] = ` "# Lazurio - + ## Tenhle Environment - Preset: \`hosted-organization-steward\` (verze 1). - Název: \`workspace\` (workspace-vm). @@ -438,7 +438,7 @@ MCP servery přicházejí na řadu až po CLI z katalogu: zjisti ve svém harnes exports[`rendered AGENTS.md snapshot: hosted-organization-steward / en 1`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-organization-steward\` (version 1). - Name: \`workspace\` (workspace-vm). @@ -492,7 +492,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`relationships render only when the recorded binding carries them, one line per peer 1`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-personal\` (version 1). - Name: \`example\` (personal-vm). @@ -545,7 +545,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`the Assignment line renders the handover's owner.assignment and nothing else 1`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-organization-personal\` (version 1). - Name: \`workspace\` (workspace-vm). @@ -595,7 +595,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`the Assignment line renders the handover's owner.assignment and nothing else 2`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-organization-team\` (version 1). - Name: \`workspace\` (workspace-vm). @@ -646,7 +646,7 @@ The complete agent manual for this Environment is in \`manual/\` (generated by t exports[`the Owner line names the Team only under hosted-organization-team 1`] = ` "# Lazurio - + ## This Environment - Preset: \`hosted-organization-personal\` (version 1). - Name: \`workspace\` (workspace-vm). diff --git a/tests/browser-panel-view.test.ts b/tests/browser-panel-view.test.ts index 27e841983..d264c57d9 100644 --- a/tests/browser-panel-view.test.ts +++ b/tests/browser-panel-view.test.ts @@ -23,16 +23,16 @@ import { import { bindings } from "./fixtures/machine-bindings"; // The Launchpad's right panel (decision F38's addendum of 2026-10-05, root -// decision 0191 point 8a, #207): offered only where the recorded entry routes -// the Environment browser's view, never on a workstation or in Recovery mode; -// it asks this origin for the view of every window and embeds it only on -// exactly the recorded origin; the address, with its token, is never kept -// beyond the open panel. The DOM (browser-panel.ts) only draws this. +// decision 0191 points 8a and 18, #207): offered only where the recorded entry +// routes the Environment browser's view, never on a workstation or in +// Recovery mode; it asks this origin for the view without a session, which is +// the people's view of a new remote tab (decision F39), and embeds it only on +// exactly the recorded origin. The DOM (browser-panel.ts) only draws this. const origin = "https://browser.workspace.example.lazurio.io"; const page = "https://launchpad.workspace.example.lazurio.io"; -const token = "0123456789abcdef".repeat(4); -const view = browserViewUrl(origin, token, null); +const target = "0123456789ABCDEF".repeat(2); +const view = browserViewUrl(origin, null); // The page's view of the recorded entry, as the server projects it and the // page parses `GET /api/entry`. @@ -73,12 +73,12 @@ test("offered only where the recorded entry routes the view: never on a workstat }); test("the view only as an https URL on exactly the recorded origin, as written: the server's own answers pass, nothing else does", async () => { - // What the server answers for the panel (no session: every window) and for - // a thread's link (a session's window): both are the view. + // What the server answers for the panel (no session: a new tab) and for a + // thread's link (a session's tab): both are the view. expect(browserViewAddress({ available: true, view }, origin, page)).toBe( view, ); - const windowView = browserViewUrl(origin, token, 42791); + const windowView = browserViewUrl(origin, target); expect( browserViewAddress( { available: true, view: windowView, session: "t3-a" }, @@ -86,19 +86,10 @@ test("the view only as an https URL on exactly the recorded origin, as written: page, ), ).toBe(windowView); - // The server's resolution itself, from the fixture's recorded entry and a - // stand-in agent-browser that answers its dashboard's URL. + // The server's resolution itself, from the fixture's recorded entry. const entry = browserEntryOf(bindings.organizationBrowser.entry); const answer = await resolveBrowserView(entry, null, { - run: async () => ({ - exitCode: 0, - stdout: JSON.stringify({ - success: true, - data: { access_urls: [`${origin}/#dashboard-access-token=${token}`] }, - }), - }), - fetch: async () => Response.json([]), - env: { HOME: "/home/operator" }, + openWindow: async () => ({ targetId: target }), }); expect(browserViewAddress(answer, origin, page)).toBe(view); @@ -115,11 +106,8 @@ test("the view only as an https URL on exactly the recorded origin, as written: { available: true, view: "javascript:alert(1)" }, // Not https, another origin, a sibling app, a port, credentials. { available: true, view: view.replace("https:", "http:") }, - { - available: true, - view: `https://elsewhere.example/#dashboard-access-token=${token}`, - }, - { available: true, view: `${page}/#dashboard-access-token=${token}` }, + { available: true, view: "https://elsewhere.example/" }, + { available: true, view: `${page}/` }, { available: true, view: view.replace(origin, `${origin}:8443`) }, { available: true, view: view.replace("https://", "https://user:secret@") }, // Not as written: the browser would read another address. @@ -158,7 +146,7 @@ test("the read: this origin's answer for every window, with the session cookie o }); expect(ok.calls).toHaveLength(1); const [call] = ok.calls; - // No session: the view of every window. No token or other header: the + // No session: a new remote tab. No token or other header: the // gateway's session cookie is the credential, sent by the browser itself. expect(call?.input).toBe("/.lazurio/browser.json"); expect(call?.init).toMatchObject({ @@ -188,7 +176,7 @@ test("the read: this origin's answer for every window, with the session cookie o () => Response.json({ available: true, - view: `https://elsewhere.example/#dashboard-access-token=${token}`, + view: "https://elsewhere.example/", }), () => { throw new TypeError("Failed to fetch: redirect mode is set to error"); diff --git a/tests/environment-browser-view-client.test.ts b/tests/environment-browser-view-client.test.ts new file mode 100644 index 000000000..eec23ba84 --- /dev/null +++ b/tests/environment-browser-view-client.test.ts @@ -0,0 +1,709 @@ +import { describe, expect, test } from "bun:test"; +import { + buttonBit, + clampInto, + clickCount, + containRect, + heldButton, + inputAction, + type KeyInput, + keyAction, + mapPoint, + modifiersOf, + mouseButton, + textMessages, + viewSize, + wheelDelta, +} from "../src/browser/people/input-map"; +import { + addressOf, + type ClientMessage, + decodeFrame, + encodeFrame, + type FrameHeader, + isTargetId, + modifierBits, + parseClientMessage, +} from "../src/browser/people/protocol"; + +// The people's view of the Environment browser (decision F39): what the view +// page sends for the person's pointer, keys and clipboard (point 6), and the +// protocol it shares with the service. Every message the page builds must be +// one the service accepts (`parseClientMessage`), or the person's input is +// silently dropped. + +const accepted = (message: ClientMessage | null) => + message !== null && parseClientMessage(JSON.stringify(message)) !== null; + +// A remote viewport of 800 × 500 CSS pixels in a frame of twice the pixels. +const header: FrameHeader = { + width: 1600, + height: 1000, + deviceWidth: 800, + deviceHeight: 500, + offsetTop: 0, + pageScaleFactor: 1, +}; + +describe("the frame in the page area", () => { + test("is drawn as large as fits, centred, with bars on the rest", () => { + expect( + containRect({ width: 1000, height: 500 }, { width: 1600, height: 1000 }), + ).toEqual({ left: 100, top: 0, width: 800, height: 500 }); + expect( + containRect({ width: 400, height: 800 }, { width: 1600, height: 1000 }), + ).toEqual({ left: 0, top: 275, width: 400, height: 250 }); + expect( + containRect({ width: 400, height: 800 }, { width: 0, height: 0 }), + ).toEqual({ left: 0, top: 0, width: 0, height: 0 }); + }); + + test("maps the pointer to the remote viewport's CSS pixels", () => { + // The page area starts 40 pixels down, under the bar. + const drawn = { left: 0, top: 40, width: 800, height: 500 }; + expect(mapPoint({ x: 400, y: 290 }, drawn, header)).toEqual({ + x: 400, + y: 250, + }); + expect(mapPoint({ x: 0, y: 40 }, drawn, header)).toEqual({ x: 0, y: 0 }); + // Drawn at half the size (an agent's viewport larger than the person's + // area): the same remote place under the same part of the frame. + const half = { left: 0, top: 40, width: 400, height: 250 }; + expect(mapPoint({ x: 200, y: 165 }, half, header)).toEqual({ + x: 400, + y: 250, + }); + }); + + test("maps through the letterbox and nothing on its bars", () => { + const area = { left: 0, top: 40, width: 1000, height: 500 }; + const inner = containRect(area, header); + const drawn = { + left: area.left + inner.left, + top: area.top + inner.top, + width: inner.width, + height: inner.height, + }; + expect(mapPoint({ x: 500, y: 290 }, drawn, header)).toEqual({ + x: 400, + y: 250, + }); + expect(mapPoint({ x: 50, y: 290 }, drawn, header)).toBeNull(); + expect(mapPoint({ x: 950, y: 290 }, drawn, header)).toBeNull(); + expect(mapPoint({ x: 500, y: 30 }, drawn, header)).toBeNull(); + expect( + mapPoint( + { x: 1, y: 1 }, + { left: 0, top: 0, width: 0, height: 0 }, + header, + ), + ).toBeNull(); + }); + + test("subtracts the remote page's top offset", () => { + const drawn = { left: 0, top: 0, width: 800, height: 500 }; + expect( + mapPoint({ x: 10, y: 100 }, drawn, { ...header, offsetTop: 20 }), + ).toEqual({ x: 10, y: 80 }); + }); + + test("a drag that leaves the frame moves along its edge", () => { + const drawn = { left: 100, top: 40, width: 800, height: 500 }; + const edge = clampInto({ x: 1200, y: 0 }, drawn); + expect(edge).toEqual({ x: 900, y: 40 }); + expect(mapPoint(edge, drawn, header)).toEqual({ x: 800, y: 0 }); + }); +}); + +const key = ( + input: Partial & Pick, +): KeyInput => ({ + type: "keydown", + ctrlKey: false, + metaKey: false, + altKey: false, + shiftKey: false, + altGraph: false, + location: 0, + repeat: false, + isComposing: false, + ...input, +}); + +describe("keys", () => { + test("punctuation keeps its virtual-key code and types itself", () => { + // Lazurio/LazurioPlatform#218: an ASCII code as the virtual-key code + // made '.' (46) arrive as Delete. + for (const [character, code, keyCode] of [ + [".", "Period", 190], + ["-", "Minus", 189], + ["'", "Quote", 222], + ] as const) { + const action = keyAction(key({ key: character, code, keyCode }), false); + expect(action).toEqual({ + send: { + t: "key", + type: "keyDown", + key: character, + code, + keyCode, + modifiers: 0, + location: 0, + repeat: false, + text: character, + }, + preventDefault: true, + }); + expect(accepted(action.send)).toBe(true); + } + }); + + test("Shift types the shifted character", () => { + const action = keyAction( + key({ key: "!", code: "Digit1", keyCode: 49, shiftKey: true }), + false, + ); + expect(action.send).toMatchObject({ + keyCode: 49, + text: "!", + modifiers: modifierBits.shift, + }); + }); + + test("a shortcut is sent without text", () => { + const action = keyAction( + key({ key: "a", code: "KeyA", keyCode: 65, ctrlKey: true }), + false, + ); + expect(action.preventDefault).toBe(true); + expect(action.send).toMatchObject({ + type: "keyDown", + key: "a", + keyCode: 65, + modifiers: modifierBits.ctrl, + }); + expect(action.send).not.toHaveProperty("text"); + expect(accepted(action.send)).toBe(true); + }); + + test("on a Mac ⌘ is the remote's Ctrl, the ⌘ key included", () => { + const selectAll = keyAction( + key({ key: "a", code: "KeyA", keyCode: 65, metaKey: true }), + true, + ); + expect(selectAll.send).toMatchObject({ modifiers: modifierBits.ctrl }); + expect(selectAll.send).not.toHaveProperty("text"); + const command = keyAction( + key({ key: "Meta", code: "MetaLeft", keyCode: 91, metaKey: true }), + true, + ); + expect(command.send).toMatchObject({ + key: "Control", + code: "ControlLeft", + keyCode: 17, + modifiers: modifierBits.ctrl, + }); + // Elsewhere the Windows or Super key stays Meta. + expect( + keyAction( + key({ key: "a", code: "KeyA", keyCode: 65, metaKey: true }), + false, + ).send, + ).toMatchObject({ modifiers: modifierBits.meta }); + }); + + test("the person's paste and copy are left to the person's browser", () => { + const pass = { send: null, preventDefault: false }; + const v = { key: "v", code: "KeyV", keyCode: 86 }; + const c = { key: "c", code: "KeyC", keyCode: 67 }; + expect(keyAction(key({ ...v, metaKey: true }), true)).toEqual(pass); + expect(keyAction(key({ ...v, ctrlKey: true }), false)).toEqual(pass); + expect( + keyAction(key({ ...v, ctrlKey: true, shiftKey: true }), false), + ).toEqual(pass); + expect(keyAction(key({ ...c, metaKey: true }), true)).toEqual(pass); + expect(keyAction(key({ ...c, ctrlKey: true }), false)).toEqual(pass); + // A layout without Latin letters: the virtual-key code decides. + expect(keyAction(key({ ...v, key: "м", ctrlKey: true }), false)).toEqual( + pass, + ); + // Ctrl+V on a Mac is no paste of the person's: the remote page gets it. + expect(keyAction(key({ ...v, ctrlKey: true }), true).send).toMatchObject({ + key: "v", + modifiers: modifierBits.ctrl, + }); + }); + + test("cut goes to the remote page and to the person's browser", () => { + const action = keyAction( + key({ key: "x", code: "KeyX", keyCode: 88, metaKey: true }), + true, + ); + expect(action.preventDefault).toBe(false); + expect(action.send).toMatchObject({ + type: "keyDown", + key: "x", + modifiers: modifierBits.ctrl, + }); + expect(action.send).not.toHaveProperty("text"); + }); + + test("Enter types a carriage return", () => { + expect( + keyAction(key({ key: "Enter", code: "Enter", keyCode: 13 }), false).send, + ).toMatchObject({ keyCode: 13, text: "\r" }); + }); + + test("a character beyond ASCII types itself", () => { + for (const character of ["ě", "ř", "😀"]) + expect( + keyAction(key({ key: character, code: "Digit2", keyCode: 50 }), false) + .send, + ).toMatchObject({ text: character }); + }); + + test("AltGr types its character, without the Ctrl that would stop it", () => { + // Windows reports AltGr as Ctrl+Alt; on a Czech layout AltGr+V is "@", + // which is neither a paste nor a shortcut. + const action = keyAction( + key({ + key: "@", + code: "KeyV", + keyCode: 86, + ctrlKey: true, + altKey: true, + altGraph: true, + }), + false, + ); + expect(action.preventDefault).toBe(true); + expect(action.send).toMatchObject({ key: "@", text: "@", modifiers: 0 }); + expect(accepted(action.send)).toBe(true); + }); + + test("a composition and a phone's keys are left to the input events", () => { + const pass = { send: null, preventDefault: false }; + expect( + keyAction( + key({ key: "a", code: "KeyA", keyCode: 65, isComposing: true }), + false, + ), + ).toEqual(pass); + expect( + keyAction(key({ key: "Process", code: "KeyA", keyCode: 229 }), false), + ).toEqual(pass); + expect( + keyAction(key({ key: "Dead", code: "Equal", keyCode: 187 }), false), + ).toEqual(pass); + expect( + keyAction(key({ key: "Unidentified", code: "", keyCode: 0 }), false), + ).toEqual(pass); + }); + + test("a key up never types", () => { + for (const input of [ + { key: "a", code: "KeyA", keyCode: 65 }, + { key: "Enter", code: "Enter", keyCode: 13 }, + ]) { + const action = keyAction(key({ ...input, type: "keyup" }), false); + expect(action.send).toMatchObject({ type: "keyUp", key: input.key }); + expect(action.send).not.toHaveProperty("text"); + expect(accepted(action.send)).toBe(true); + } + }); + + test("keys that type nothing carry no text", () => { + for (const input of [ + { key: "Tab", code: "Tab", keyCode: 9 }, + { key: "Backspace", code: "Backspace", keyCode: 8 }, + { key: "ArrowLeft", code: "ArrowLeft", keyCode: 37 }, + ]) + expect(keyAction(key(input), false).send).not.toHaveProperty("text"); + }); + + test("modifier bits", () => { + const none = { ctrlKey: false, metaKey: false, altKey: false }; + expect(modifiersOf({ ...none, shiftKey: true }, false)).toBe( + modifierBits.shift, + ); + expect(modifiersOf({ ...none, altKey: true, shiftKey: false }, true)).toBe( + modifierBits.alt, + ); + expect( + modifiersOf( + { ctrlKey: true, metaKey: true, altKey: false, shiftKey: false }, + true, + ), + ).toBe(modifierBits.ctrl); + }); +}); + +describe("text", () => { + test("a paste keeps its text, with line breaks as \\n", () => { + expect(textMessages("a\r\nb\rc\nd")).toEqual([ + { t: "text", text: "a\nb\nc\nd" }, + ]); + expect(textMessages("")).toEqual([]); + }); + + test("a long text comes in parts the service accepts", () => { + const messages = textMessages("x".repeat(250_001)); + expect( + messages.map((message) => (message.t === "text" ? message.text : "")), + ).toEqual(["x".repeat(100_000), "x".repeat(100_000), "x".repeat(50_001)]); + expect(messages.every(accepted)).toBe(true); + }); + + test("a part never splits a character", () => { + expect(textMessages("a😀b", 2)).toEqual([ + { t: "text", text: "a" }, + { t: "text", text: "😀" }, + { t: "text", text: "b" }, + ]); + }); + + test("a phone keyboard's edits", () => { + expect(inputAction("insertText", "ahoj")).toEqual([ + { t: "text", text: "ahoj" }, + ]); + expect(inputAction("insertReplacementText", "word")).toEqual([ + { t: "text", text: "word" }, + ]); + expect(inputAction("insertText", null)).toEqual([]); + const backspace = inputAction("deleteContentBackward", null); + expect(backspace).toEqual([ + { + t: "key", + type: "keyDown", + key: "Backspace", + code: "Backspace", + keyCode: 8, + modifiers: 0, + location: 0, + repeat: false, + }, + { + t: "key", + type: "keyUp", + key: "Backspace", + code: "Backspace", + keyCode: 8, + modifiers: 0, + location: 0, + repeat: false, + }, + ]); + for (const inputType of ["insertParagraph", "insertLineBreak"]) { + const enter = inputAction(inputType, null); + expect(enter?.[0]).toMatchObject({ + type: "keyDown", + key: "Enter", + keyCode: 13, + text: "\r", + }); + expect(enter?.[1]).toMatchObject({ type: "keyUp", key: "Enter" }); + expect(enter?.[1]).not.toHaveProperty("text"); + expect(enter?.every(accepted)).toBe(true); + } + expect(backspace?.every(accepted)).toBe(true); + // A composition's text is sent whole when it ends. + expect(inputAction("insertCompositionText", "ahoj")).toBeNull(); + }); +}); + +describe("the pointer", () => { + test("wheel lines and pages become pixels", () => { + expect(wheelDelta({ deltaX: 3, deltaY: -40, deltaMode: 0 }, 500)).toEqual({ + deltaX: 3, + deltaY: -40, + }); + expect(wheelDelta({ deltaX: 0, deltaY: 3, deltaMode: 1 }, 500)).toEqual({ + deltaX: 0, + deltaY: 48, + }); + expect(wheelDelta({ deltaX: 0, deltaY: -1, deltaMode: 2 }, 500)).toEqual({ + deltaX: 0, + deltaY: -500, + }); + expect( + wheelDelta({ deltaX: 1e9, deltaY: -1e9, deltaMode: 0 }, 500), + ).toEqual({ deltaX: 100_000, deltaY: -100_000 }); + }); + + test("buttons", () => { + expect([0, 1, 2, 3, 4, 5, -1].map(mouseButton)).toEqual([ + "left", + "middle", + "right", + "back", + "forward", + "none", + "none", + ]); + // A move drags with the button held (DOM `buttons`). + expect([0, 1, 2, 4, 3, 8, 16].map(heldButton)).toEqual([ + "none", + "left", + "right", + "middle", + "left", + "back", + "forward", + ]); + expect([0, 1, 2, 3, 4, 7].map(buttonBit)).toEqual([1, 4, 2, 8, 16, 0]); + }); + + test("double and triple clicks are counted by time, place and button", () => { + const first = { time: 1000, x: 10, y: 10, button: "left" as const }; + expect(clickCount(null, first)).toBe(1); + const once = { ...first, count: 1 }; + expect(clickCount(once, { ...first, time: 1300, x: 12 })).toBe(2); + expect(clickCount({ ...once, count: 2 }, { ...first, time: 1300 })).toBe(3); + expect(clickCount(once, { ...first, time: 1600 })).toBe(1); + expect(clickCount(once, { ...first, time: 1100, x: 20 })).toBe(1); + expect(clickCount(once, { ...first, time: 1100, button: "right" })).toBe(1); + }); + + test("every pointer message the page builds is accepted", () => { + const at = mapPoint( + { x: 400, y: 290 }, + { left: 0, top: 40, width: 800, height: 500 }, + header, + ); + expect(at).not.toBeNull(); + const point = at ?? { x: 0, y: 0 }; + const messages: ClientMessage[] = [ + { + t: "mouse", + type: "mousePressed", + ...point, + button: mouseButton(0), + buttons: 1, + clickCount: clickCount(null, { time: 0, x: 0, y: 0, button: "left" }), + modifiers: modifiersOf( + { ctrlKey: false, metaKey: true, altKey: false, shiftKey: true }, + true, + ), + }, + { + t: "mouse", + type: "mouseWheel", + ...point, + button: "none", + buttons: 0, + clickCount: 0, + modifiers: 0, + ...wheelDelta({ deltaX: 0, deltaY: 1e9, deltaMode: 0 }, 500), + }, + { + t: "touch", + type: "touchStart", + points: [{ id: 0, ...point }], + modifiers: 0, + }, + { t: "touch", type: "touchEnd", points: [], modifiers: 0 }, + ]; + expect(messages.every(accepted)).toBe(true); + }); +}); + +describe("the size of the page area", () => { + test("is whole pixels within the protocol's limits", () => { + expect(viewSize(1280.6, 720.4, 2)).toEqual({ + width: 1281, + height: 720, + dpr: 2, + }); + expect(viewSize(100, 100, 0.5)).toEqual({ + width: 200, + height: 150, + dpr: 1, + }); + expect(viewSize(5000, 3000, 4)).toEqual({ + width: 3840, + height: 2400, + dpr: 3, + }); + expect(viewSize(Number.NaN, Number.NaN, Number.NaN)).toEqual({ + width: 200, + height: 150, + dpr: 1, + }); + expect( + accepted({ t: "hello", ...viewSize(390, 640, 3), resize: true }), + ).toBe(true); + }); +}); + +describe("the protocol", () => { + const jpeg = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 0xff, 0xd9]); + + test("a frame survives the socket", () => { + const frame = encodeFrame(header, jpeg); + expect(decodeFrame(frame)).toEqual({ header, jpeg }); + // The page receives an ArrayBuffer. + const received = decodeFrame(new Uint8Array(frame).buffer); + expect(received?.header).toEqual(header); + expect(received?.jpeg).toEqual(jpeg); + }); + + test("a truncated or malformed frame is refused", () => { + const frame = encodeFrame(header, jpeg); + const headLength = frame.length - jpeg.length - 4; + expect(decodeFrame(frame.subarray(0, 4 + headLength - 1))).toBeNull(); + expect(decodeFrame(frame.subarray(0, 3))).toBeNull(); + expect(decodeFrame(new Uint8Array(0))).toBeNull(); + const huge = new Uint8Array(frame); + new DataView(huge.buffer).setUint32(0, 5000, false); + expect(decodeFrame(huge)).toBeNull(); + const notJson = new Uint8Array([0, 0, 0, 2, 0x7b, 0x7b]); + expect(decodeFrame(notJson)).toBeNull(); + const head = new TextEncoder().encode( + JSON.stringify({ ...header, deviceWidth: "800" }), + ); + const wrong = new Uint8Array(4 + head.length); + new DataView(wrong.buffer).setUint32(0, head.length, false); + wrong.set(head, 4); + expect(decodeFrame(wrong)).toBeNull(); + }); + + test("every kind of message of the page is accepted", () => { + const messages: ClientMessage[] = [ + { t: "hello", width: 1280, height: 720, dpr: 2, resize: true }, + { t: "size", width: 390, height: 640, dpr: 3 }, + { + t: "mouse", + type: "mouseMoved", + x: 1, + y: 2, + button: "none", + buttons: 0, + clickCount: 0, + modifiers: 0, + }, + { + t: "mouse", + type: "mouseWheel", + x: 1, + y: 2, + button: "none", + buttons: 0, + clickCount: 0, + modifiers: 0, + deltaX: 0, + deltaY: 120, + }, + { + t: "key", + type: "keyDown", + key: "a", + code: "KeyA", + keyCode: 65, + text: "a", + modifiers: 0, + location: 0, + repeat: false, + }, + { + t: "key", + type: "keyUp", + key: "a", + code: "KeyA", + keyCode: 65, + modifiers: 0, + location: 0, + repeat: false, + }, + { t: "text", text: "ahoj" }, + { + t: "touch", + type: "touchMove", + points: [ + { id: 0, x: 1, y: 2 }, + { id: 1, x: 3, y: 4 }, + ], + modifiers: 0, + }, + { t: "touch", type: "touchCancel", points: [], modifiers: 0 }, + { t: "nav", action: "back" }, + { t: "nav", action: "stop" }, + { t: "goto", url: "https://example.com/" }, + { t: "dialog", accept: true, text: "jméno" }, + { t: "dialog", accept: false }, + { t: "files-cancel", token: "0123456789abcdef01234567" }, + { t: "close" }, + ]; + for (const message of messages) + expect(parseClientMessage(JSON.stringify(message))).toEqual(message); + }); + + test("anything else is refused", () => { + const mouse = { + t: "mouse", + type: "mousePressed", + x: 1, + y: 2, + button: "left", + buttons: 1, + clickCount: 1, + modifiers: 0, + }; + const touch = { t: "touch", type: "touchStart", modifiers: 0 }; + for (const raw of [ + "not json", + "[]", + "null", + JSON.stringify({ t: "launch" }), + JSON.stringify({ t: "text", text: "x".repeat(1_000_001) }), + JSON.stringify({ t: "text", text: "" }), + JSON.stringify({ t: "hello", width: 1280, height: 720, dpr: 2 }), + JSON.stringify({ ...mouse, x: null }), + JSON.stringify(mouse).replace('"x":1', '"x":NaN'), + JSON.stringify({ ...mouse, x: 1e9 }), + JSON.stringify({ ...mouse, button: "thumb" }), + JSON.stringify({ ...touch, points: [{ x: 1, y: 2 }] }), + JSON.stringify({ ...touch, points: [{ id: 0, x: "1", y: 2 }] }), + JSON.stringify({ ...touch, points: "0,1,2" }), + JSON.stringify({ + ...touch, + points: Array.from({ length: 11 }, (_, id) => ({ id, x: 1, y: 2 })), + }), + JSON.stringify({ + t: "key", + type: "keyDown", + key: "a", + code: "KeyA", + keyCode: 65, + text: "", + modifiers: 0, + location: 0, + repeat: false, + }), + JSON.stringify({ t: "nav", action: "home" }), + ]) + expect(parseClientMessage(raw)).toBeNull(); + }); + + test("a tab is named by its DevTools target id only", () => { + expect(isTargetId("0123456789ABCDEF0123456789abcdef")).toBe(true); + for (const value of ["0123", "../../etc/passwd", "g".repeat(32), 42]) + expect(isTargetId(value)).toBe(false); + }); + + test("the address bar's text becomes an address or a search", () => { + expect(addressOf("example.com")).toBe("https://example.com/"); + expect(addressOf(" localhost:3000 ")).toBe("http://localhost:3000/"); + expect(addressOf("http://x.y/z")).toBe("http://x.y/z"); + expect(addressOf("about:blank")).toBe("about:blank"); + expect(addressOf("hello world")).toBe( + "https://www.google.com/search?q=hello%20world", + ); + // Never a script or a file of the Environment: a search for the text. + expect(addressOf("javascript:alert(1)")).toBe( + `https://www.google.com/search?q=${encodeURIComponent("javascript:alert(1)")}`, + ); + expect(addressOf("file:///etc/passwd")).toBe( + `https://www.google.com/search?q=${encodeURIComponent("file:///etc/passwd")}`, + ); + }); +}); diff --git a/tests/environment-browser-view.test.ts b/tests/environment-browser-view.test.ts new file mode 100644 index 000000000..abd09f093 --- /dev/null +++ b/tests/environment-browser-view.test.ts @@ -0,0 +1,1013 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { CdpConnection, CdpEvent } from "../src/browser/people/cdp"; +import { + browserExtensionFiles, + writeBrowserExtension, +} from "../src/browser/people/extension"; +import { + BrowserHub, + jpegSize, + navigableAddress, + type Viewer, +} from "../src/browser/people/hub"; +import { + type ClientMessage, + decodeFrame, + type ServerMessage, +} from "../src/browser/people/protocol"; +import { parseServeArgs } from "../src/browser/people/serve"; +import { + classifyRequest, + frameAncestors, + pageHeaders, + startViewService, + type ViewService, +} from "../src/browser/people/service"; + +// The people's view of the Environment browser (decision F39) against a +// scripted DevTools peer: no Chrome runs, every call is recorded and +// answered the way Chrome answers it. + +const origin = "https://browser.pilot.example.lazurio.io"; +const host = new URL(origin).host; + +// The smallest bytes `jpegSize` reads: SOI and a baseline SOF0 of 3×2. +const jpeg = Buffer.from([ + 0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x00, 0x02, 0x00, 0x03, 0x03, 0x01, + 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, 0xff, 0xd9, +]); + +type Call = { + method: string; + params: Record; + sessionId?: string; +}; +type Target = { + targetId: string; + url: string; + title: string; + openerId?: string; + canAccessOpener?: boolean; +}; + +const id = (n: number) => n.toString(16).toUpperCase().padStart(32, "0"); + +/** A browser target that answers like Chrome for what the hub asks. */ +class FakeBrowser { + readonly targets = new Map(); + readonly calls: Call[] = []; + private listeners: ((event: CdpEvent) => void)[] = []; + private closers: (() => void)[] = []; + private next = 100; + connections = 0; + open = false; + /** What the page's selection and title read as. */ + selection = ""; + pageTitle = "Stránka z dokumentu"; + + constructor(initial: Target[] = []) { + for (const target of initial) this.targets.set(target.targetId, target); + } + + connect = async (): Promise => { + this.connections++; + this.open = true; + this.listeners = []; + this.closers = []; + return { + send: async (method, params = {}, sessionId) => { + if (!this.open) throw new Error("closed"); + this.calls.push({ + method, + params: { ...params }, + ...(sessionId === undefined ? {} : { sessionId }), + }); + return this.answer( + method, + params as Record, + sessionId, + ); + }, + onEvent: (listener) => { + this.listeners.push(listener); + }, + onClose: (listener) => { + this.closers.push(listener); + }, + close: () => this.drop(), + }; + }; + + drop(): void { + if (!this.open) return; + this.open = false; + for (const closer of this.closers.splice(0)) closer(); + } + + emit(method: string, params: Record, sessionId?: string) { + for (const listener of this.listeners) + listener({ method, params, ...(sessionId ? { sessionId } : {}) }); + } + + created(target: Target): void { + this.targets.set(target.targetId, target); + this.emit("Target.targetCreated", { + targetInfo: { type: "page", attached: false, ...target }, + }); + } + + called(method: string): Call[] { + return this.calls.filter((call) => call.method === method); + } + + private answer( + method: string, + params: Record, + sessionId?: string, + ): Record { + switch (method) { + case "Target.setDiscoverTargets": + for (const target of this.targets.values()) + this.emit("Target.targetCreated", { + targetInfo: { type: "page", attached: false, ...target }, + }); + return {}; + case "Target.attachToTarget": + return { sessionId: `S-${params.targetId}` }; + case "Target.createTarget": { + const targetId = id(this.next++); + queueMicrotask(() => + this.created({ targetId, url: String(params.url), title: "" }), + ); + return { targetId }; + } + case "Target.closeTarget": + this.targets.delete(String(params.targetId)); + queueMicrotask(() => + this.emit("Target.targetDestroyed", { targetId: params.targetId }), + ); + return { success: true }; + case "Page.getFrameTree": + return { frameTree: { frame: { id: `F-${sessionId}` } } }; + case "Page.getNavigationHistory": + return { currentIndex: 1, entries: [{ id: 1 }, { id: 2 }, { id: 3 }] }; + case "Page.captureScreenshot": + return { data: jpeg.toString("base64") }; + case "Page.getLayoutMetrics": + return { + cssVisualViewport: { clientWidth: 1280, clientHeight: 800 }, + cssLayoutViewport: { clientWidth: 1280, clientHeight: 800 }, + }; + case "Browser.getWindowForTarget": + return { windowId: 7, bounds: { width: 1280, height: 885 } }; + case "Page.createIsolatedWorld": + return { executionContextId: 3 }; + case "Runtime.evaluate": + return { + result: { + value: String(params.expression).includes("getSelection") + ? this.selection + : JSON.stringify({ + title: this.pageTitle, + icon: "https://example.com/favicon.ico", + }), + }, + }; + default: + return {}; + } + } +} + +const settle = (ms = 20) => new Promise((resolve) => setTimeout(resolve, ms)); +async function until(check: () => boolean, ms = 2_000): Promise { + const deadline = Date.now() + ms; + while (!check()) { + if (Date.now() > deadline) throw new Error("condition not reached"); + await settle(5); + } +} + +let root: string | undefined; +let service: ViewService | undefined; +afterEach(async () => { + await service?.stop(); + service = undefined; + if (root) await rm(root, { recursive: true, force: true }); + root = undefined; +}); + +async function serve( + browser: FakeBrowser, + options: { bound?: string[]; graceMs?: number } = {}, +) { + root = await realpath(await mkdtemp(join(tmpdir(), "people-view-"))); + const logs: string[] = []; + service = startViewService({ + origin, + port: 0, + log: (line) => logs.push(line), + hub: { + connect: browser.connect, + boundTargets: async () => new Set(options.bound ?? []), + uploadDirectory: join(root, "uploads"), + screen: { width: 1920, height: 1080 }, + log: (line) => logs.push(line), + graceMs: options.graceMs ?? 60_000, + unviewedGraceMs: options.graceMs ?? 60_000, + reconnectMs: 20, + }, + }); + await until(() => service?.hub.connected === true); + const base = `http://127.0.0.1:${service.server.port}`; + return { base, logs, root }; +} + +/** A person's socket, as the view page opens it. */ +async function person( + base: string, + targetId: string, + headers = { Host: host, Origin: origin }, +) { + const messages: ( + | ServerMessage + | { frame: ReturnType } + )[] = []; + const socket = new WebSocket( + `${base.replace("http", "ws")}/t/${targetId}/live`, + { + // Bun's client sends these as given, as the gateway forwards them. + headers, + } as unknown as string[], + ); + socket.binaryType = "arraybuffer"; + socket.addEventListener("message", (event) => { + if (typeof event.data === "string") + messages.push(JSON.parse(event.data) as ServerMessage); + else messages.push({ frame: decodeFrame(event.data as ArrayBuffer) }); + }); + const closed = new Promise((resolve) => + socket.addEventListener("close", () => resolve()), + ); + await new Promise((resolve, reject) => { + socket.addEventListener("open", () => resolve()); + socket.addEventListener("error", () => reject(new Error("socket error"))); + }); + const send = (message: ClientMessage) => socket.send(JSON.stringify(message)); + const of = (t: K) => + messages.filter( + (m): m is Extract => "t" in m && m.t === t, + ); + return { socket, messages, send, of, closed }; +} + +const tab = (n: number, extra: Partial = {}): Target => ({ + targetId: id(n), + url: "https://example.com/", + title: "Example", + ...extra, +}); + +// ---- Admission (F39 point 8) ------------------------------------------------- + +test("every request must name the view's host; sockets and changes must come from its origin", () => { + const ask = ( + method: string, + path: string, + headers: { + host?: string | null; + origin?: string | null; + upgrade?: boolean; + }, + ) => + classifyRequest( + { + method, + url: `http://127.0.0.1:4848${path}`, + host: headers.host === undefined ? host : headers.host, + origin: headers.origin ?? null, + upgrade: headers.upgrade ?? false, + }, + origin, + 4848, + ); + const live = `/t/${id(1)}/live`; + expect(ask("GET", "/", {})).toEqual({ kind: "page" }); + expect(ask("GET", `/t/${id(1)}`, {})).toEqual({ kind: "page" }); + expect(ask("GET", "/assets/view.js", {})).toEqual({ kind: "script" }); + expect(ask("GET", "/", { host: "evil.example" }).kind).toBe("refused"); + // DNS rebinding names another host. + expect( + ask("GET", live, { host: "127.0.0.1:4848", upgrade: true, origin }), + ).toEqual({ + kind: "refused", + status: 421, + error: "wrong-host", + }); + expect(ask("GET", live, { upgrade: true, origin })).toEqual({ + kind: "live", + targetId: id(1), + }); + // Another app of the same site is same-site but not this origin. + for (const other of [ + null, + "https://t3code.pilot.example.lazurio.io", + "https://browser.other.example.lazurio.io", + ]) + expect(ask("GET", live, { upgrade: true, origin: other })).toEqual({ + kind: "refused", + status: 403, + error: "wrong-origin", + }); + expect(ask("POST", "/api/tabs", { origin })).toEqual({ kind: "create" }); + expect( + ask("POST", "/api/tabs", { origin: "https://evil.example" }).kind, + ).toBe("refused"); + expect(ask("GET", "/api/tabs", { origin }).kind).toBe("refused"); + expect(ask("POST", `/t/${id(1)}/files`, {}).kind).toBe("refused"); + expect(ask("POST", "/", { origin }).kind).toBe("refused"); + expect(ask("GET", "/json/version", {}).kind).toBe("refused"); + // Doctor asks the health on loopback. + expect(ask("GET", "/.lazurio/health", { host: "127.0.0.1:4848" })).toEqual({ + kind: "health", + }); + expect(ask("GET", "/.lazurio/health", { host: "evil.example" }).kind).toBe( + "refused", + ); +}); + +test("only the Environment's own origins may frame the view", () => { + expect(frameAncestors(origin)).toBe( + "'self' https://*.pilot.example.lazurio.io", + ); + expect(frameAncestors("https://browser.someone.lazurio.io")).toBe( + "'self' https://*.someone.lazurio.io", + ); + expect(frameAncestors("http://localhost:4848")).toBe("'self'"); + const csp = pageHeaders(origin)["content-security-policy"] ?? ""; + expect(csp).toContain("script-src 'self'"); + expect(csp).toContain(`connect-src 'self' wss://${host}`); + expect(csp).toContain( + "frame-ancestors 'self' https://*.pilot.example.lazurio.io", + ); +}); + +test("the service refuses another host and a socket from another origin end to end", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const wrongHost = await fetch(`${base}/`, { + headers: { Host: "evil.example" }, + }); + expect(wrongHost.status).toBe(421); + const page = await fetch(`${base}/t/${id(1)}`, { headers: { Host: host } }); + expect(page.status).toBe(200); + expect(page.headers.get("content-security-policy")).toContain( + "frame-ancestors", + ); + const health = await fetch(`${base}/.lazurio/health`); + expect(await health.json()).toEqual({ ok: true, browser: "connected" }); + await expect( + person(base, id(1), { + Host: host, + Origin: "https://t3code.pilot.example.lazurio.io", + }), + ).rejects.toThrow(); +}); + +// ---- Tabs (F39 points 1 and 2) ----------------------------------------------- + +test("a person's new tab is a new window of the default context at the person's size", async () => { + const browser = new FakeBrowser(); + const { base } = await serve(browser); + const created = await fetch(`${base}/api/tabs`, { + method: "POST", + headers: { Host: host, Origin: origin, "content-type": "application/json" }, + body: JSON.stringify({ width: 900, height: 600, dpr: 2 }), + }); + expect(created.status).toBe(201); + const body = (await created.json()) as { id: string; view: string }; + expect(body.view).toBe(`/t/${body.id}`); + expect(browser.called("Target.createTarget")[0]?.params).toEqual({ + url: "about:blank", + newWindow: true, + width: 900, + height: 600, + }); + expect(service?.hub.hasTab(body.id)).toBe(true); +}); + +test("a viewer gets the tab's state, frames at its size, and Chrome is acked", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + expect(view.of("browser")[0]).toEqual({ t: "browser", state: "connected" }); + view.send({ t: "hello", width: 700, height: 500, dpr: 2, resize: true }); + await until(() => browser.called("Page.startScreencast").length > 0); + expect(browser.called("Page.startScreencast")[0]?.params).toMatchObject({ + format: "jpeg", + maxWidth: 1400, + maxHeight: 1000, + }); + browser.emit( + "Page.screencastFrame", + { + data: jpeg.toString("base64"), + sessionId: 5, + metadata: { + deviceWidth: 700, + deviceHeight: 500, + offsetTop: 0, + pageScaleFactor: 1, + }, + }, + `S-${id(1)}`, + ); + await until(() => browser.called("Page.screencastFrameAck").length > 0); + expect(browser.called("Page.screencastFrameAck")[0]?.params).toEqual({ + sessionId: 5, + }); + const frames = () => + view.messages.flatMap((m) => ("frame" in m && m.frame ? [m.frame] : [])); + // The frame reaches the person after Chrome is acked. + await until(() => frames().some((f) => f.header.deviceWidth === 700)); + const last = frames().at(-1); + expect(last?.header).toEqual({ + width: 3, + height: 2, + deviceWidth: 700, + deviceHeight: 500, + offsetTop: 0, + pageScaleFactor: 1, + }); + // F39 point 3: the window takes the person's page area plus Chrome's bars. + await until(() => browser.called("Browser.setWindowBounds").length > 0); + expect(browser.called("Browser.setWindowBounds")[0]?.params).toEqual({ + windowId: 7, + bounds: { width: 700, height: 585 }, + }); + // Every page has its dialogs and file choosers kept in the view. + expect( + browser + .called("Page.setInterceptFileChooserDialog") + .map((c) => c.sessionId), + ).toContain(`S-${id(1)}`); + view.socket.close(); +}); + +test("the title and icon come from the page itself, since Chrome reports a target's title only as its address", async () => { + const browser = new FakeBrowser([ + tab(1, { url: "https://example.com/", title: "example.com/" }), + ]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => + view.of("info").some((m) => m.title === "Stránka z dokumentu"), + ); + expect(view.of("info").at(-1)).toMatchObject({ + url: "https://example.com/", + title: "Stránka z dokumentu", + favicon: "https://example.com/favicon.ico", + canGoBack: true, + canGoForward: true, + }); + // A later change of the page's title follows while someone views it. + browser.pageTitle = "(1) Nová zpráva"; + await until( + () => view.of("info").some((m) => m.title === "(1) Nová zpráva"), + 4_000, + ); + view.socket.close(); +}); + +// ---- Input (F39 points 5 and 6) ---------------------------------------------- + +test("keys keep their codes, paste inserts text, the mouse lands where it was sent", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + view.send({ + t: "key", + type: "keyDown", + key: ".", + code: "Period", + keyCode: 190, + text: ".", + modifiers: 0, + location: 0, + repeat: false, + }); + view.send({ + t: "key", + type: "keyDown", + key: "a", + code: "KeyA", + keyCode: 65, + modifiers: 2, + location: 0, + repeat: false, + }); + view.send({ t: "text", text: "heslo z mé schránky" }); + view.send({ + t: "mouse", + type: "mousePressed", + x: 120.5, + y: 80, + button: "left", + buttons: 1, + clickCount: 1, + modifiers: 0, + }); + await until(() => browser.called("Input.dispatchMouseEvent").length > 0); + const keys = browser.called("Input.dispatchKeyEvent").map((c) => c.params); + expect(keys[0]).toMatchObject({ + type: "keyDown", + key: ".", + code: "Period", + windowsVirtualKeyCode: 190, + text: ".", + }); + expect(keys[1]).toMatchObject({ type: "rawKeyDown", key: "a", modifiers: 2 }); + expect(keys[1]?.text).toBeUndefined(); + expect(browser.called("Input.insertText")[0]?.params).toEqual({ + text: "heslo z mé schránky", + }); + expect(browser.called("Input.dispatchMouseEvent")[0]?.params).toMatchObject({ + type: "mousePressed", + x: 120.5, + y: 80, + button: "left", + }); + // Co-control (F39 point 5): nothing locks the agent out. + expect( + browser.calls.some((c) => /lock|Emulation.setTouch/.test(c.method)), + ).toBe(false); + view.socket.close(); +}); + +test("the address bar navigates only to web addresses", async () => { + expect(navigableAddress("https://example.com/a")).toBe( + "https://example.com/a", + ); + expect(navigableAddress("about:blank")).toBe("about:blank"); + expect(navigableAddress("javascript:alert(1)")).toBeNull(); + expect(navigableAddress("file:///etc/passwd")).toBeNull(); + expect(navigableAddress("chrome://settings")).toBeNull(); + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + view.send({ t: "goto", url: "javascript:alert(1)" }); + view.send({ t: "goto", url: "https://example.org/" }); + view.send({ t: "nav", action: "back" }); + await until(() => browser.called("Page.navigateToHistoryEntry").length > 0); + expect(browser.called("Page.navigate").map((c) => c.params.url)).toEqual([ + "https://example.org/", + ]); + expect(browser.called("Page.navigateToHistoryEntry")[0]?.params).toEqual({ + entryId: 1, + }); + view.socket.close(); +}); + +test("the remote selection reaches the person for copy and cut, read when it may have changed", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + browser.selection = "zkopírovaný text"; + // A drag that ends selects; the release is when the selection is read. + view.send({ + t: "mouse", + type: "mouseReleased", + x: 10, + y: 10, + button: "left", + buttons: 0, + clickCount: 1, + modifiers: 0, + }); + await until(() => view.of("selection").length > 0); + expect(view.of("selection")[0]).toEqual({ + t: "selection", + text: "zkopírovaný text", + }); + // Ctrl (or ⌘) pressed, just before a copy, reads it again. + browser.selection = "jiný výběr"; + view.send({ + t: "key", + type: "keyDown", + key: "Control", + code: "ControlLeft", + keyCode: 17, + modifiers: 2, + location: 1, + repeat: false, + }); + await until(() => + view.of("selection").some((message) => message.text === "jiný výběr"), + ); + // Read in the view's isolated world; the Runtime domain is never enabled, + // which pages could detect as automation. + expect(browser.called("Page.createIsolatedWorld")[0]?.params).toMatchObject({ + worldName: "lazurio-view", + }); + expect(browser.called("Runtime.enable")).toHaveLength(0); + expect(browser.called("Runtime.addBinding")).toHaveLength(0); + view.socket.close(); +}); + +// ---- Pop-ups and new tabs (F39 point 4) -------------------------------------- + +test("a page that can reach its opener is a pop-up; one that cannot is a new tab", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + browser.created( + tab(2, { + url: "https://accounts.google.com/o/oauth2", + openerId: id(1), + canAccessOpener: true, + }), + ); + browser.created( + tab(3, { + url: "https://example.com/doc", + openerId: id(1), + canAccessOpener: false, + }), + ); + browser.created(tab(4, { openerId: id(9), canAccessOpener: true })); + await until( + () => view.of("popup").length > 0 && view.of("new-tab").length > 0, + ); + expect(view.of("popup")).toEqual([ + { t: "popup", id: id(2), url: "https://accounts.google.com/o/oauth2" }, + ]); + expect(view.of("new-tab")).toEqual([ + { t: "new-tab", id: id(3), url: "https://example.com/doc" }, + ]); + view.socket.close(); +}); + +// ---- Nothing outside the page blocks the view (F39 point 7) ------------------ + +test("a page's dialog is answered from the view", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + browser.emit( + "Page.javascriptDialogOpening", + { type: "prompt", message: "Jméno?", defaultPrompt: "Matěj" }, + `S-${id(1)}`, + ); + await until(() => view.of("dialog").length > 0); + expect(view.of("dialog")[0]).toEqual({ + t: "dialog", + kind: "prompt", + message: "Jméno?", + defaultPrompt: "Matěj", + }); + view.send({ t: "dialog", accept: true, text: "Anička" }); + await until(() => browser.called("Page.handleJavaScriptDialog").length > 0); + expect(browser.called("Page.handleJavaScriptDialog")[0]?.params).toEqual({ + accept: true, + promptText: "Anička", + }); + view.socket.close(); +}); + +test("a file chooser is answered with the person's files; without a viewer it is dropped", async () => { + const browser = new FakeBrowser([tab(1), tab(2)]); + const { base, logs } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + browser.emit( + "Page.fileChooserOpened", + { frameId: "F", mode: "selectSingle", backendNodeId: 42 }, + `S-${id(1)}`, + ); + await until(() => view.of("files").length > 0); + const { token, multiple } = view.of("files")[0] as { + token: string; + multiple: boolean; + }; + expect(multiple).toBe(false); + const form = new FormData(); + form.append("file", new File(["obsah faktury"], "faktura ../x.pdf")); + const refused = await fetch(`${base}/t/${id(1)}/files?token=${token}`, { + method: "POST", + headers: { Host: host }, + body: form, + }); + expect(refused.status).toBe(403); + const uploaded = await fetch(`${base}/t/${id(1)}/files?token=${token}`, { + method: "POST", + headers: { Host: host, Origin: origin }, + body: form, + }); + expect(uploaded.status).toBe(204); + const set = browser.called("DOM.setFileInputFiles")[0]; + if (set === undefined) throw new Error("no setFileInputFiles"); + expect(set.params.backendNodeId).toBe(42); + const [path] = set.params.files as string[]; + // Only the base name survives, so a name cannot leave the upload folder. + expect(path?.endsWith("/0/x.pdf")).toBe(true); + expect(await readFile(path as string, "utf8")).toBe("obsah faktury"); + await until(() => view.of("files-closed").length > 0); + // The token is single-use. + const again = await fetch(`${base}/t/${id(1)}/files?token=${token}`, { + method: "POST", + headers: { Host: host, Origin: origin }, + body: form, + }); + expect(again.status).toBe(409); + browser.emit( + "Page.fileChooserOpened", + { frameId: "F", mode: "selectSingle", backendNodeId: 43 }, + `S-${id(2)}`, + ); + await settle(); + expect( + logs.some((line) => line.includes("file chooser") && line.includes(id(2))), + ).toBe(true); + view.socket.close(); +}); + +// ---- Closing (F39 point 9) --------------------------------------------------- + +test("a closed person's tab closes its own remote tab after the grace, unless an agent is bound to it", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser, { graceMs: 60, bound: [] }); + const make = async () => { + const response = await fetch(`${base}/api/tabs`, { + method: "POST", + headers: { + Host: host, + Origin: origin, + "content-type": "application/json", + }, + body: "{}", + }); + return ((await response.json()) as { id: string }).id; + }; + const own = await make(); + const view = await person(base, own); + await until(() => view.of("info").length > 0); + view.socket.close(); + await until(() => + browser.called("Target.closeTarget").some((c) => c.params.targetId === own), + ); + // An agent's tab the view did not open is never closed by it. + const agent = await person(base, id(1)); + await until(() => agent.of("info").length > 0); + agent.socket.close(); + await settle(150); + expect( + browser + .called("Target.closeTarget") + .some((c) => c.params.targetId === id(1)), + ).toBe(false); +}); + +test("an own tab an agent bound itself to stays open; an own tab nobody opened closes", async () => { + const browser = new FakeBrowser(); + // The fake's first two new tabs are id(100) and id(101); an agent-browser + // session is bound to the first. + const { base } = await serve(browser, { graceMs: 40, bound: [id(100)] }); + const make = async () => { + const response = await fetch(`${base}/api/tabs`, { + method: "POST", + headers: { + Host: host, + Origin: origin, + "content-type": "application/json", + }, + body: "{}", + }); + return ((await response.json()) as { id: string }).id; + }; + const bound = await make(); + expect(bound).toBe(id(100)); + const view = await person(base, bound); + await until(() => view.of("info").length > 0); + view.socket.close(); + const unopened = await make(); + await until(() => + browser + .called("Target.closeTarget") + .some((c) => c.params.targetId === unopened), + ); + await settle(100); + expect( + browser + .called("Target.closeTarget") + .some((c) => c.params.targetId === bound), + ).toBe(false); +}); + +test("a tab that goes away tells its viewers", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + browser.emit("Target.targetDestroyed", { targetId: id(1) }); + await view.closed; + expect(view.of("closed")).toEqual([{ t: "closed" }]); + // A socket for a tab that does not exist is refused before it opens. + await expect(person(base, id(77))).rejects.toThrow(); +}); + +test("a lost browser connection is reported and the view resumes on its tab", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + browser.drop(); + await until(() => view.of("browser").some((m) => m.state === "reconnecting")); + await until( + () => browser.connections >= 2 && service?.hub.connected === true, + ); + await until( + () => view.of("browser").filter((m) => m.state === "connected").length >= 2, + ); + expect( + browser + .called("Target.attachToTarget") + .filter((c) => c.params.targetId === id(1)).length, + ).toBeGreaterThanOrEqual(2); + view.socket.close(); +}); + +// ---- Flow control ------------------------------------------------------------ + +test("a slow viewer holds Chrome's next frame instead of the service's memory", async () => { + const browser = new FakeBrowser([tab(1)]); + const hub = new BrowserHub({ + connect: browser.connect, + boundTargets: async () => new Set(), + uploadDirectory: tmpdir(), + screen: { width: 1920, height: 1080 }, + log: () => {}, + reconnectMs: 20, + }); + hub.start(); + await until(() => hub.connected); + let backlog = 2 << 20; + const viewer: Viewer = { + send: () => {}, + sendFrame: () => {}, + buffered: () => backlog, + close: () => {}, + size: null, + resize: true, + }; + await until(() => hub.attach(id(1), viewer)); + await hub.handle(id(1), viewer, { + t: "hello", + width: 800, + height: 600, + dpr: 1, + resize: false, + }); + await until(() => browser.called("Page.startScreencast").length > 0); + browser.emit( + "Page.screencastFrame", + { + data: jpeg.toString("base64"), + sessionId: 9, + metadata: { deviceWidth: 800, deviceHeight: 600 }, + }, + `S-${id(1)}`, + ); + await settle(); + expect(browser.called("Page.screencastFrameAck")).toHaveLength(0); + backlog = 0; + hub.drained(id(1)); + await until(() => browser.called("Page.screencastFrameAck").length === 1); + // A pop-up viewer (resize false) never resizes its window. + expect(browser.called("Browser.setWindowBounds")).toHaveLength(0); + hub.stop(); +}); + +test("jpegSize reads a frame's size and refuses other bytes", () => { + expect(jpegSize(jpeg)).toEqual({ width: 3, height: 2 }); + expect(jpegSize(Buffer.from("not a jpeg"))).toBeNull(); +}); + +// ---- The extension (F39 point 7) --------------------------------------------- + +test("the extension declines passkeys and leaves other credentials alone", async () => { + const original: string[] = []; + class CredentialsContainer { + get(options: unknown) { + original.push(`get ${JSON.stringify(options)}`); + return Promise.resolve("password-credential"); + } + create(options: unknown) { + original.push(`create ${JSON.stringify(options)}`); + return Promise.resolve("created"); + } + } + const PublicKeyCredential = { + isUserVerifyingPlatformAuthenticatorAvailable: () => Promise.resolve(true), + isConditionalMediationAvailable: () => Promise.resolve(true), + }; + const scope = { CredentialsContainer, PublicKeyCredential, DOMException }; + new Function( + "globalThis", + "DOMException", + browserExtensionFiles["webauthn.js"] as string, + )(scope, DOMException); + const container = new CredentialsContainer(); + const declined = await container + .get({ publicKey: { challenge: new Uint8Array(1) } }) + .catch((error: DOMException) => error.name); + expect(declined).toBe("NotAllowedError"); + expect( + await container + .create({ publicKey: {} }) + .catch((error: DOMException) => error.name), + ).toBe("NotAllowedError"); + expect(await container.get({ password: true })).toBe("password-credential"); + expect(original).toEqual(['get {"password":true}']); + expect( + await PublicKeyCredential.isUserVerifyingPlatformAuthenticatorAvailable(), + ).toBe(false); + expect(await PublicKeyCredential.isConditionalMediationAvailable()).toBe( + false, + ); +}); + +test("the extension moves a second tab of a window into a window of its own", async () => { + const moved: unknown[] = []; + let listener: + | ((tab: { id?: number; windowId?: number }) => Promise) + | undefined; + const windows = new Map([ + [1, { type: "normal", tabs: 2 }], + [2, { type: "popup", tabs: 2 }], + [3, { type: "normal", tabs: 1 }], + ]); + const chrome = { + tabs: { + onCreated: { addListener: (fn: typeof listener) => (listener = fn) }, + query: async ({ windowId }: { windowId: number }) => + Array.from({ length: windows.get(windowId)?.tabs ?? 0 }), + }, + windows: { + get: async (windowId: number) => windows.get(windowId), + create: async (options: unknown) => moved.push(options), + }, + }; + new Function("chrome", browserExtensionFiles["background.js"] as string)( + chrome, + ); + await listener?.({ id: 10, windowId: 1 }); + await listener?.({ id: 11, windowId: 2 }); + await listener?.({ id: 12, windowId: 3 }); + expect(moved).toEqual([{ tabId: 10, focused: false }]); +}); + +test("the extension is written next to the profile and only when it differs", async () => { + root = await realpath(await mkdtemp(join(tmpdir(), "extension-"))); + expect(await writeBrowserExtension(root)).toBe(true); + expect(await writeBrowserExtension(root)).toBe(false); + const manifest = JSON.parse( + await readFile( + join(root, ".local/share/lazurio-browser/extension/manifest.json"), + "utf8", + ), + ) as { + permissions?: unknown; + content_scripts: { world: string; run_at: string }[]; + }; + // No permission at all, and the passkey script runs before the page. + expect(manifest.permissions).toBeUndefined(); + expect(manifest.content_scripts[0]).toMatchObject({ + world: "MAIN", + run_at: "document_start", + }); +}); + +test("serve takes exactly a loopback port and an origin", () => { + expect(parseServeArgs(["--port", "4848", "--origin", origin])).toEqual({ + port: 4848, + origin, + }); + expect( + parseServeArgs(["--port", "4848", "--origin", `${origin}/`]), + ).toBeNull(); + expect(parseServeArgs(["--port", "0", "--origin", origin])).toBeNull(); + expect(parseServeArgs(["--port", "4848"])).toBeNull(); + expect(parseServeArgs(["--port", "4848", "--origin", "ftp://x"])).toBeNull(); + expect( + parseServeArgs(["extra", "--port", "4848", "--origin", origin]), + ).toBeNull(); +}); diff --git a/tests/environment-browser.test.ts b/tests/environment-browser.test.ts index 43488a452..27aef4e9d 100644 --- a/tests/environment-browser.test.ts +++ b/tests/environment-browser.test.ts @@ -16,6 +16,7 @@ import { browserLink, runBrowserCommand, } from "../src/browser/cli"; +import { browserExtensionDigest } from "../src/browser/people/extension"; import { browserCdpPort, browserUnit, @@ -57,14 +58,16 @@ import { import organizationContext from "./fixtures/machine-context.json"; import { commitOf, target } from "./fixtures/update-world"; -// The Environment browser (root decision 0191, decision F38): its units, -// their observation, the view's link, a thread's window and the CLI. No real -// systemd, browser or agent-browser: every command is recorded and answered. +// The Environment browser (root decision 0191, decisions F38 and F39): its +// units, their observation, the view's link, a thread's window and the CLI. +// No real systemd, browser or agent-browser: every command is recorded and +// answered. The people's view service itself is tested in +// environment-browser-view.test.ts. const origin = "https://browser.workspace.example.lazurio.io"; const { team: _team, ...withoutTeamOwner } = organizationContext.owner; const entry = Object.freeze({ origin, listenPort: 4848 }); -const token = "a".repeat(64); +const selector = "/home/operator/.local/share/lazurio/bin/lazurio"; let root: string | undefined; afterEach(async () => { @@ -78,9 +81,11 @@ async function temporary(prefix: string) { // ---- Units ------------------------------------------------------------------- -// The exact unit texts proven on a Remote Environment on 2026-10-05: Xvfb -// waits for its socket, Chrome is the browser unit's main process, the -// dashboard detaches from its oneshot unit. +// The unit texts: the screen and the browser as proven on a Remote +// Environment on 2026-10-05 (Xvfb waits for its socket, Chrome is the browser +// unit's main process), the browser with F39's flags and its extension's +// digest, and the people's view as a long-running service of this +// installation. const displayLines = [ unitMarker, "[Unit]", @@ -108,7 +113,8 @@ const browserLines = [ "[Service]", "Type=simple", "Environment=DISPLAY=:1", - 'ExecStart=/bin/sh -c \'b=$$(ls -d %h/.agent-browser/browsers/chrome-*/chrome %h/.agent-browser/browsers/chrome-*/chrome-linux64/chrome 2>/dev/null | sort -V | tail -n 1); [ -x "$$b" ] || exit 78; exec "$$b" --user-data-dir=%h/.local/share/lazurio-browser/profile --remote-debugging-port=9222 --no-first-run --no-default-browser-check --password-store=basic --disable-background-timer-throttling --disable-renderer-backgrounding --disable-backgrounding-occluded-windows --hide-crash-restore-bubble --window-size=1280,900 about:blank\'', + `Environment=LAZURIO_BROWSER_EXTENSION=${browserExtensionDigest}`, + 'ExecStart=/bin/sh -c \'b=$$(ls -d %h/.agent-browser/browsers/chrome-*/chrome %h/.agent-browser/browsers/chrome-*/chrome-linux64/chrome 2>/dev/null | sort -V | tail -n 1); [ -x "$$b" ] || exit 78; exec "$$b" --user-data-dir=%h/.local/share/lazurio-browser/profile --remote-debugging-port=9222 --no-first-run --no-default-browser-check --password-store=basic --disable-background-timer-throttling --disable-renderer-backgrounding --disable-backgrounding-occluded-windows --hide-crash-restore-bubble --window-size=1280,900 --deny-permission-prompts --load-extension=%h/.local/share/lazurio-browser/extension --disable-features=DisableLoadExtensionCommandLineSwitch about:blank\'', "Restart=always", "RestartSec=3", "RestartPreventExitStatus=78", @@ -118,24 +124,21 @@ const browserLines = [ "", ]; -test("the three units render the texts proven on a Remote Environment", () => { +test("the three units render the screen, the browser with its extension and the people's view", () => { expect(renderDisplayUnit().split("\n")).toEqual(displayLines); expect(renderBrowserUnit().split("\n")).toEqual(browserLines); - expect(renderBrowserViewUnit(entry).split("\n")).toEqual([ + expect(renderBrowserViewUnit(entry, selector).split("\n")).toEqual([ unitMarker, "[Unit]", - "Description=Lazurio Environment browser view: the agent-browser dashboard behind the gateway", - "ConditionFileIsExecutable=%h/.local/bin/agent-browser", + "Description=Lazurio Environment browser view: one tab of a person is one tab of the Environment browser", + "After=lazurio-browser.service", "", "[Service]", - "Type=oneshot", - "RemainAfterExit=yes", - "KillMode=process", + "Type=simple", "Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin", - "Environment=AGENT_BROWSER_CDP=9222", - `ExecStart=%h/.local/bin/agent-browser dashboard start --port 4848 --allowed-origins ${origin}`, - "ExecStop=-%h/.local/bin/agent-browser dashboard stop", - "TimeoutStartSec=60", + `ExecStart=${selector} browser serve --port 4848 --origin ${origin}`, + "Restart=always", + "RestartSec=2", "", "[Install]", "WantedBy=default.target", @@ -155,7 +158,7 @@ function recorder(failing: string[] = []) { return { commands, run }; } -test("installed only for a hosted operator whose handover routes the view; never stops the screen or the browser", async () => { +test("installed only for a hosted operator whose handover routes the view; the browser restarts only when its own unit changed", async () => { const directory = join(await temporary("browser-units-"), "units"); await mkdir(directory, { recursive: true }); const { commands, run } = recorder(); @@ -166,6 +169,7 @@ test("installed only for a hosted operator whose handover routes the view; never directory, hosted: false, entry, + selector, run, env, }), @@ -175,6 +179,7 @@ test("installed only for a hosted operator whose handover routes the view; never directory, hosted: true, entry: undefined, + selector, run, env, }), @@ -187,6 +192,7 @@ test("installed only for a hosted operator whose handover routes the view; never directory, hosted: true, entry, + selector, run, env, }), @@ -204,9 +210,14 @@ test("installed only for a hosted operator whose handover routes the view; never browserUnit, browserViewUnit, ], + ["systemctl", "--user", "try-restart", browserUnit], ["systemctl", "--user", "try-restart", browserViewUnit], ["systemctl", "--user", "start", displayUnit, browserUnit, browserViewUnit], ]); + // The extension is written next to the profile, never into it. + expect( + await readdir(join(root ?? "", ".local/share/lazurio-browser/extension")), + ).toEqual(["background.js", "manifest.json", "webauthn.js"]); const written = await stat(join(directory, browserUnit)); // Repeated with the same entry: nothing rewritten, no reload, no restart; @@ -217,6 +228,7 @@ test("installed only for a hosted operator whose handover routes the view; never directory, hosted: true, entry, + selector, run, env, }), @@ -244,6 +256,7 @@ test("installed only for a hosted operator whose handover routes the view; never directory, hosted: true, entry: { origin, listenPort: 4849 }, + selector, run, env, }), @@ -271,6 +284,7 @@ test("a unit someone else wrote is left alone, and a refusing manager is a findi directory, hosted: true, entry, + selector, run, env, }); @@ -294,6 +308,7 @@ test("a unit someone else wrote is left alone, and a refusing manager is a findi directory, hosted: true, entry, + selector, run: failing.run, env, }), @@ -387,142 +402,46 @@ test("doctor: skipped where the browser cannot be, a warning naming the unit or // ---- The view ---------------------------------------------------------------- -function dashboard( - answer: unknown, - sessions: unknown = [], -): Readonly<{ run: ProcessRunner; fetch: typeof fetch; calls: string[][] }> { - const calls: string[][] = []; - const run: ProcessRunner = async (command) => { - calls.push([...command]); - return answer === null - ? { exitCode: 1, stdout: "" } - : { exitCode: 0, stdout: JSON.stringify(answer) }; - }; - const fetcher = (async (url: string, init?: RequestInit) => { - calls.push(["fetch", url, new Headers(init?.headers).get("origin") ?? ""]); - return Response.json(sessions); - }) as typeof fetch; - return { run, fetch: fetcher, calls }; -} -const started = (urls: unknown) => ({ - success: true, - data: { access_urls: urls, port: 4848 }, -}); - -// A thread asks for its own window before its agent opened one (the Browser +// A thread asks for its own tab before its agent opened one (the Browser // panel of a new T3 Code thread): the view opens the thread's window first, -// so the person and the agent see the same window. Found on the first -// Environment: the panel showed the list of every window and the person -// followed another session's window while the agent worked in the thread's. -test("the view of a thread without a window opens that window first, and falls back to every window when it cannot", async () => { - const env = { HOME: "/home/operator", XDG_RUNTIME_DIR: "/run/user/1000" }; - const sessions: { engine: string; port: number; session: string }[] = [ - { engine: "chrome", port: 42337, session: "other" }, - ]; - const ok = dashboard( - started([`${origin}/#dashboard-access-token=${token}`]), - sessions, - ); +// so the person and the agent see the same tab. Without a session (the +// Launchpad's panel) it is a new remote tab: the view of every window is +// retired (root decision 0191 point 18, decision F39). +test("the view of a thread is its own tab, opened when it has none; without a session a new tab; nothing when the window cannot be opened", async () => { const opened: string[] = []; const openWindow = async (session: string) => { opened.push(session); - sessions.push({ engine: "chrome", port: 45229, session }); + return { targetId: "A".repeat(32) }; }; const thread = "t3-3745367c-d453-41df-b948-75e29eff8651"; - expect( - await resolveBrowserView(entry, thread, { ...ok, env, openWindow }), - ).toEqual({ + expect(await resolveBrowserView(entry, thread, { openWindow })).toEqual({ available: true, - view: `${origin}/?port=45229&view=.html#dashboard-access-token=${token}`, + view: `${origin}/t/${"A".repeat(32)}`, session: thread, }); expect(opened).toEqual([thread]); - // Its window runs now: asked again, nothing is opened. - await resolveBrowserView(entry, thread, { ...ok, env, openWindow }); - expect(opened).toEqual([thread]); - // Every window (no session) never opens one. - await resolveBrowserView(entry, null, { ...ok, env, openWindow }); + // No session: a new remote tab, and no window is opened for it here. + expect(await resolveBrowserView(entry, null, { openWindow })).toEqual({ + available: true, + view: `${origin}/`, + session: null, + }); expect(opened).toEqual([thread]); - // The window cannot be opened: the view of every window, as before. + // The window cannot be opened: unavailable, never another tab. const failing = async () => { throw new Error("browser-unreachable"); }; expect( - await resolveBrowserView(entry, "t3-another", { - ...ok, - env, - openWindow: failing, - }), - ).toEqual({ - available: true, - view: `${origin}/#dashboard-access-token=${token}`, - session: "t3-another", - }); -}); - -test("the view: the dashboard's token from agent-browser's own answer, the session's window by its stream port", async () => { - const env = { HOME: "/home/operator", XDG_RUNTIME_DIR: "/run/user/1000" }; - const ok = dashboard( - started([`${origin}/#dashboard-access-token=${token}`]), - [ - { engine: "chrome", port: 42791, session: "codex-thread" }, - { engine: "chrome", port: 42337, session: "other" }, - ], - ); - expect( - await resolveBrowserView(entry, "codex-thread", { ...ok, env }), - ).toEqual({ - available: true, - view: `${origin}/?port=42791&view=.html#dashboard-access-token=${token}`, - session: "codex-thread", - }); - // Exactly the unit's settings, on the Environment's own agent-browser, and - // the session list asked on loopback with a loopback Origin. - expect(ok.calls[0]).toEqual([ - "/home/operator/.local/bin/agent-browser", - "dashboard", - "start", - "--port", - "4848", - "--allowed-origins", - origin, - "--json", - ]); - expect(ok.calls[1]).toEqual([ - "fetch", - "http://127.0.0.1:4848/api/sessions", - "http://127.0.0.1:4848", - ]); - // A session that is not running yet, or none named: every window. - expect(await resolveBrowserView(entry, "unknown", { ...ok, env })).toEqual({ - available: true, - view: `${origin}/#dashboard-access-token=${token}`, - session: "unknown", - }); - expect(await resolveBrowserView(entry, null, { ...ok, env })).toMatchObject({ - available: true, - session: null, - }); - // No entry, no agent-browser, another origin or a token of another shape. - expect(await resolveBrowserView(undefined, null, { ...ok, env })).toEqual({ + await resolveBrowserView(entry, "t3-another", { openWindow: failing }), + ).toEqual({ available: false, reason: "view-unavailable" }); + expect(await resolveBrowserView(undefined, null, { openWindow })).toEqual({ available: false, reason: "not-declared", }); - for (const answer of [ - null, - started([]), - started([`https://elsewhere.example/#dashboard-access-token=${token}`]), - started([`${origin}/#dashboard-access-token=xyz`]), - { success: true }, - ]) { - const refused = dashboard(answer); - expect(await resolveBrowserView(entry, null, { ...refused, env })).toEqual({ - available: false, - reason: "view-unavailable", - }); - } - expect(browserViewUrl(origin, token, null)).toBe( - `${origin}/#dashboard-access-token=${token}`, + // No address carries a token any more. + expect(browserViewUrl(origin, null)).toBe(`${origin}/`); + expect(browserViewUrl(origin, "B".repeat(32))).toBe( + `${origin}/t/${"B".repeat(32)}`, ); expect(browserEntryOf(null)).toBeUndefined(); expect( @@ -586,7 +505,7 @@ function windowSeams( return { seams, calls }; } -test("a thread's window: the bound one while it is open, otherwise a new window of the shared context bound and pinned", async () => { +test("a thread's window: the bound one while it is open, otherwise a new window of the shared context bound and pinned without a fixed viewport", async () => { const socketDir = await temporary("browser-window-"); // Bound and open: reused, nothing created. await writeFile( @@ -602,7 +521,9 @@ test("a thread's window: the bound one while it is open, otherwise a new window expect(reused.calls).toEqual([]); // Bound but closed, or never bound: a new window in the default context - // (no browserContextId), then agent-browser's own tab binding and pin. + // (no browserContextId), then agent-browser's own tab binding and a pin by + // a command that changes nothing: the people's view sets the page size + // (F39 point 3). const created = windowSeams({ socketDir, targets: [] }); expect( await ensureThreadWindow("codex-a", "https://example.com", created.seams), @@ -629,10 +550,8 @@ test("a thread's window: the bound one while it is open, otherwise a new window "--session", "codex-a", "--pin-tab", - "set", - "viewport", - "1280", - "800", + "get", + "url", ], ]); @@ -774,17 +693,20 @@ test.skipIf(process.platform === "win32")( session: "codex-019a", targetId: "NEWTARGET", created: true, - link: `${launchpad}/.lazurio/browser?session=codex-019a`, + link: `${origin}/t/NEWTARGET`, command: "agent-browser --cdp 9222 --session codex-019a", }); - expect(browserLink(launchpad, null)).toBe(`${launchpad}/.lazurio/browser`); + expect(browserLink(launchpad, origin, null)).toBe(`${origin}/`); + expect(browserLink(launchpad, origin, "codex-019a")).toBe( + `${launchpad}/.lazurio/browser?session=codex-019a`, + ); expect(browserCommand("x")).toBe("agent-browser --cdp 9222 --session x"); const link = await runBrowserCommand( ["link", "--folder", folder], context({ HOME: home }), ); - expect(link).toEqual({ code: 0, stdout: `${launchpad}/.lazurio/browser` }); + expect(link).toEqual({ code: 0, stdout: `${origin}/` }); // No session from the thread and none named: a usage answer. expect( @@ -852,10 +774,6 @@ async function hostedLaunchpad(withView: boolean) { "__Secure-lazurio-workspace=valid" ? new Response("ok") : new Response("no", { status: 401 }); - const view = dashboard( - started([`${origin}/#dashboard-access-token=${token}`]), - [{ engine: "chrome", port: 42791, session: "codex-a" }], - ); const app = await startLaunchpad( folder, undefined, @@ -874,7 +792,12 @@ async function hostedLaunchpad(withView: boolean) { undefined, undefined, undefined, - { run: view.run, fetch: view.fetch, env: { HOME: "/home/operator" } }, + { + openWindow: async (session: string) => { + if (session !== "codex-a") throw new Error("browser-unreachable"); + return { targetId: "C".repeat(32) }; + }, + }, ); const base = `http://127.0.0.1:${listenPort}`; const valid = { @@ -907,7 +830,7 @@ test.skipIf(process.platform === "win32")( const answer = await hosted.get("/.lazurio/browser.json?session=codex-a"); expect(answer.status).toBe(200); expect(answer.headers.get("cache-control")).toBe("no-store"); - const view = `${origin}/?port=42791&view=.html#dashboard-access-token=${token}`; + const view = `${origin}/t/${"C".repeat(32)}`; expect(await answer.json()).toEqual({ available: true, view, @@ -919,10 +842,14 @@ test.skipIf(process.platform === "win32")( view, ]); expect(handOver.headers.get("referrer-policy")).toBe("no-referrer"); - // Every window without a session. + // A new remote tab without a session. + expect(await (await hosted.get("/.lazurio/browser.json")).json()).toEqual( + { available: true, view: `${origin}/`, session: null }, + ); + // A thread whose window cannot be opened: unavailable, not found. expect( - await (await hosted.get("/.lazurio/browser.json")).json(), - ).toMatchObject({ available: true, session: null }); + (await hosted.get("/.lazurio/browser?session=codex-b")).status, + ).toBe(404); // Only `session`, only agent-browser's alphabet, only GET, only // admitted. for (const path of [ diff --git a/tests/folder-template-revision.test.ts b/tests/folder-template-revision.test.ts index cf3aec6d8..459222cfc 100644 --- a/tests/folder-template-revision.test.ts +++ b/tests/folder-template-revision.test.ts @@ -21,8 +21,8 @@ import { bindings } from "./fixtures/machine-bindings"; // `bun test -u` does not touch it. The fixtures in `tests/fixtures` feed the // compositions too. const pinned = { - revision: "base-instructions-30", - digest: "48530dbd122335ffd0154bba0e4d411987f1eae3602dd13bdd6286b41741ea7b", + revision: "base-instructions-31", + digest: "78b996f73a836ae0bb6911e15f1f11949b5602ced3974806623fb61ca1c38283", } as const; const optionalTools = activatableTools() diff --git a/tests/folder-tools.test.ts b/tests/folder-tools.test.ts index 8b6c5e94e..2ca214cba 100644 --- a/tests/folder-tools.test.ts +++ b/tests/folder-tools.test.ts @@ -519,7 +519,7 @@ test("a profile change and a handover refresh carry the enabled tools forward", }); test("a Folder rendered by an older template revision is upgraded by a tools change", async () => { - expect(instructionTemplateRevision).toBe("base-instructions-30"); + expect(instructionTemplateRevision).toBe("base-instructions-31"); const { preferences, manifest } = await planned(); const older = { ...manifest, @@ -542,7 +542,7 @@ test("a Folder rendered by an older template revision is upgraded by a tools cha recorded, ); if (upgrade.kind !== "profile-change") throw new Error("Expected upgrade"); - expect(upgrade.manifest.templateRevision).toBe("base-instructions-30"); + expect(upgrade.manifest.templateRevision).toBe("base-instructions-31"); expect(upgrade.previous).toEqual(older.outputs); expect(enabledTools(upgrade.preferences)).toEqual(tools); expect(upgrade.files).toEqual( diff --git a/tests/update-install.test.ts b/tests/update-install.test.ts index 6a1e15df9..a7c172bc9 100644 --- a/tests/update-install.test.ts +++ b/tests/update-install.test.ts @@ -1077,7 +1077,9 @@ test("on a supervised hosted base whose handover routes the browser's view, inst "lazurio-display.service", ].sort(), ); - expect(commands.slice(-4)).toEqual([ + // A unit written now is a changed unit: the browser and the view are + // restarted if they ran (F39 point 10); `try-restart` starts nothing. + expect(commands.slice(-5)).toEqual([ ["systemctl", "--user", "daemon-reload"], [ "systemctl", @@ -1087,6 +1089,7 @@ test("on a supervised hosted base whose handover routes the browser's view, inst "lazurio-browser.service", "lazurio-browser-view.service", ], + ["systemctl", "--user", "try-restart", "lazurio-browser.service"], ["systemctl", "--user", "try-restart", "lazurio-browser-view.service"], [ "systemctl", From 9162a46e30472176b66249ebeea290c292395cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= Date: Tue, 6 Oct 2026 21:04:20 +0200 Subject: [PATCH 2/4] Environment browser view: a viewer of a tab that is gone is told so, not left reconnecting (F39) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The service refused the socket of an unknown tab before the upgrade, so the page saw only a failed handshake and retried forever with "Connecting…" (after a browser restart, every open view). It now upgrades and says `closed`, and the page offers a new tab. Only a hub that has discovered every existing page says a tab is gone: while it reconnects to the browser, a new socket is told `reconnecting` and closed with 1013, and the page tries again. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/browser/people/hub.ts | 9 +++++++++ src/browser/people/service.ts | 13 ++++++++++--- tests/environment-browser-view.test.ts | 22 ++++++++++++++++++---- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/browser/people/hub.ts b/src/browser/people/hub.ts index 1013e3113..edfe704df 100644 --- a/src/browser/people/hub.ts +++ b/src/browser/people/hub.ts @@ -200,6 +200,7 @@ export class BrowserHub { private readonly fileChoosers = new Map(); /** Page targets discovery reported on the current connection. */ private seen = new Set(); + private discovered = false; private readonly graceMs: number; private readonly unviewedGraceMs: number; private readonly reconnectMs: number; @@ -214,6 +215,12 @@ export class BrowserHub { return this.cdp !== null; } + /** Connected and discovery has reported every existing page: a target + * the hub does not know now does not exist. */ + get ready(): boolean { + return this.cdp !== null && this.discovered; + } + start(): void { this.stopped = false; void this.connect(); @@ -513,6 +520,7 @@ export class BrowserHub { this.retry(); }); this.seen = new Set(); + this.discovered = false; try { // Chrome reports every existing target before it answers. await cdp.send("Target.setDiscoverTargets", { discover: true }); @@ -527,6 +535,7 @@ export class BrowserHub { if (!this.seen.has(id)) this.gone(id); for (const id of [...this.ownTabs]) if (!this.seen.has(id)) this.ownTabs.delete(id); + this.discovered = true; } private retry(): void { diff --git a/src/browser/people/service.ts b/src/browser/people/service.ts index c554c5d24..88142a552 100644 --- a/src/browser/people/service.ts +++ b/src/browser/people/service.ts @@ -227,8 +227,8 @@ export function startViewService(options: ViewServiceOptions): ViewService { : json({ error: "not-accepted" }, 409); } case "live": { - if (!hub.hasTab(decision.targetId)) - return json({ error: "no-such-tab" }, 404); + // Upgraded even for a tab that is gone: the page can read a + // message, not the status of a refused handshake (`open`). const upgraded = server.upgrade(request, { data: { targetId: decision.targetId, viewer: null }, }); @@ -255,9 +255,16 @@ export function startViewService(options: ViewServiceOptions): ViewService { resize: true, }; ws.data.viewer = viewer; - if (!hub.attach(ws.data.targetId, viewer)) { + if (hub.attach(ws.data.targetId, viewer)) return; + // No such tab. Only a hub that knows every page may say it is gone; + // while it reconnects to the browser the page tries again. + ws.data.viewer = null; + if (hub.ready) { viewer.send({ t: "closed" }); ws.close(1000, "closed"); + } else { + viewer.send({ t: "browser", state: "reconnecting" }); + ws.close(1013, "reconnecting"); } }, message(ws: ServerWebSocket, data) { diff --git a/tests/environment-browser-view.test.ts b/tests/environment-browser-view.test.ts index abd09f093..1e842503e 100644 --- a/tests/environment-browser-view.test.ts +++ b/tests/environment-browser-view.test.ts @@ -202,7 +202,7 @@ afterEach(async () => { async function serve( browser: FakeBrowser, - options: { bound?: string[]; graceMs?: number } = {}, + options: { bound?: string[]; graceMs?: number; reconnectMs?: number } = {}, ) { root = await realpath(await mkdtemp(join(tmpdir(), "people-view-"))); const logs: string[] = []; @@ -218,7 +218,7 @@ async function serve( log: (line) => logs.push(line), graceMs: options.graceMs ?? 60_000, unviewedGraceMs: options.graceMs ?? 60_000, - reconnectMs: 20, + reconnectMs: options.reconnectMs ?? 20, }, }); await until(() => service?.hub.connected === true); @@ -820,8 +820,22 @@ test("a tab that goes away tells its viewers", async () => { browser.emit("Target.targetDestroyed", { targetId: id(1) }); await view.closed; expect(view.of("closed")).toEqual([{ t: "closed" }]); - // A socket for a tab that does not exist is refused before it opens. - await expect(person(base, id(77))).rejects.toThrow(); + // A tab that no longer exists, opened again (a reload after a browser + // restart): told so, not left reconnecting. + const stale = await person(base, id(77)); + await stale.closed; + expect(stale.of("closed")).toEqual([{ t: "closed" }]); +}); + +test("while the browser is away, a new socket is told to try again, never that its tab is gone", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser, { reconnectMs: 60_000 }); + browser.drop(); + await until(() => service?.hub.connected === false); + const view = await person(base, id(1)); + await view.closed; + expect(view.of("browser")).toEqual([{ t: "browser", state: "reconnecting" }]); + expect(view.of("closed")).toEqual([]); }); test("a lost browser connection is reported and the view resumes on its tab", async () => { From e225489a12d94947b252008b8442e8593200ce52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= Date: Tue, 6 Oct 2026 21:07:07 +0200 Subject: [PATCH 3/4] Environment browser view: clear a stale selection; no native context menu (F39) From the client's review against the service: - A navigation reset the tab's selection in the hub but not in the viewers, so a copy after a navigation still took the old page's text. The hub now sends the empty selection when a new document replaces a selected one. - A right click reached the page, and where the page drew no menu of its own, Chrome opened its native context menu outside the page, which nobody sees in the view (like the passkey window). The extension's third job turns Chrome's own menu off with a last listener on the window; a page's own menu still works. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/decisions.md | 4 +++- src/browser/people/extension.ts | 21 +++++++++++++++-- src/browser/people/hub.ts | 6 ++++- tests/environment-browser-view.test.ts | 31 +++++++++++++++++++++++++- tests/environment-browser.test.ts | 2 +- 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index f004893ab..ce5de5f68 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -5194,7 +5194,9 @@ F38 points 1, 5 and 6 as stated below; points 2–4 and 7 stand. Lazurio/LazurioPlatform#204), answers every WebAuthn request of a page (`navigator.credentials` with `publicKey`) with `NotAllowedError` at once, and reports no platform authenticator and no conditional mediation. Chrome's - native passkey window never opens, and sites offer another way. + native passkey window never opens, and sites offer another way; + - turns Chrome's own context menu off: it too would open outside the page. + A page's own menu still works. - The view server keeps a DevTools session on every page: - **JavaScript dialogs** (alert, confirm, prompt, beforeunload) show in the view and are answered from it; diff --git a/src/browser/people/extension.ts b/src/browser/people/extension.ts index 3b199e37b..d0843c9c9 100644 --- a/src/browser/people/extension.ts +++ b/src/browser/people/extension.ts @@ -5,13 +5,15 @@ import { writeDurableFile } from "../../update/durable-file"; /** The Lazurio extension of the Environment browser (decision F39 point 7), * loaded with `--load-extension` from a directory next to the profile, never - * inside it. Two jobs, no permissions: + * inside it. Three jobs, no permissions: * - a tab a page opens in a window that already has one moves into a window * of its own, because Chrome paints only visible windows and the people's * view shows one tab per person's tab (point 2); * - until the Environment's Bitwarden holds its passkeys (0191 point 17), a * WebAuthn request is declined at once, so Chrome's own passkey window, - * which the view cannot show and which blocks the page, never opens. */ + * which the view cannot show and which blocks the page, never opens; + * - Chrome's own context menu never opens either: it would sit outside the + * page, where nobody sees it. A page's own menu still works. */ export const browserExtensionDirectory = (home: string) => join(home, ".local", "share", "lazurio-browser", "extension"); @@ -33,6 +35,13 @@ const manifest = `${JSON.stringify( match_about_blank: true, world: "MAIN", }, + { + matches: [""], + js: ["menu.js"], + run_at: "document_start", + all_frames: true, + match_about_blank: true, + }, ], }, null, @@ -96,11 +105,19 @@ const webauthn = `// Lazurio (decision F39, root decision 0191 point 17): until })(); `; +const menu = `// Lazurio (decision F39 point 7): Chrome's own context menu would open +// outside the page, where the people's view cannot show it. It is turned off +// for every page; a page's own menu, which it draws itself, still works. This +// listener runs last (the window, in the bubbling phase), after the page's. +window.addEventListener("contextmenu", (event) => event.preventDefault()); +`; + export const browserExtensionFiles: Readonly> = Object.freeze({ "manifest.json": manifest, "background.js": background, "webauthn.js": webauthn, + "menu.js": menu, }); /** One digest of every file, carried in the browser unit's text, so a new diff --git a/src/browser/people/hub.ts b/src/browser/people/hub.ts index edfe704df..57e4d54cb 100644 --- a/src/browser/people/hub.ts +++ b/src/browser/people/hub.ts @@ -993,7 +993,11 @@ export class BrowserHub { if (frame?.parentId !== undefined) return; if (typeof frame?.id === "string") tab.mainFrameId = frame.id; tab.world = null; - tab.selection = ""; + // A new document has no selection: a copy must not take the old one. + if (tab.selection !== "") { + tab.selection = ""; + this.broadcast(tab, { t: "selection", text: "" }); + } void this.history(tab); return; } diff --git a/tests/environment-browser-view.test.ts b/tests/environment-browser-view.test.ts index 1e842503e..38225de7b 100644 --- a/tests/environment-browser-view.test.ts +++ b/tests/environment-browser-view.test.ts @@ -611,6 +611,13 @@ test("the remote selection reaches the person for copy and cut, read when it may await until(() => view.of("selection").some((message) => message.text === "jiný výběr"), ); + // A navigation clears it: a copy must not take the old page's text. + browser.emit( + "Page.frameNavigated", + { frame: { id: `F-S-${id(1)}`, url: "https://example.org/" } }, + `S-${id(1)}`, + ); + await until(() => view.of("selection").at(-1)?.text === ""); // Read in the view's isolated world; the Runtime domain is never enabled, // which pages could detect as automation. expect(browser.called("Page.createIsolatedWorld")[0]?.params).toMatchObject({ @@ -959,6 +966,21 @@ test("the extension declines passkeys and leaves other credentials alone", async ); }); +test("the extension turns Chrome's own context menu off and leaves a page's own menu working", () => { + const listeners: ((event: Event) => void)[] = []; + const window = { + addEventListener: (type: string, listener: (event: Event) => void) => { + if (type === "contextmenu") listeners.push(listener); + }, + }; + new Function("window", browserExtensionFiles["menu.js"] as string)(window); + const event = new Event("contextmenu", { cancelable: true }); + // The page's own handler ran before (a target or capture listener) and + // drew its menu; the extension only keeps Chrome's away. + for (const listener of listeners) listener(event); + expect(event.defaultPrevented).toBe(true); +}); + test("the extension moves a second tab of a window into a window of its own", async () => { const moved: unknown[] = []; let listener: @@ -1000,14 +1022,21 @@ test("the extension is written next to the profile and only when it differs", as ), ) as { permissions?: unknown; - content_scripts: { world: string; run_at: string }[]; + content_scripts: { js: string[]; world?: string; run_at: string }[]; }; // No permission at all, and the passkey script runs before the page. expect(manifest.permissions).toBeUndefined(); expect(manifest.content_scripts[0]).toMatchObject({ + js: ["webauthn.js"], world: "MAIN", run_at: "document_start", }); + // The menu script runs in the extension's own world, at the start too. + expect(manifest.content_scripts[1]).toMatchObject({ + js: ["menu.js"], + run_at: "document_start", + }); + expect(manifest.content_scripts[1]?.world).toBeUndefined(); }); test("serve takes exactly a loopback port and an origin", () => { diff --git a/tests/environment-browser.test.ts b/tests/environment-browser.test.ts index 27aef4e9d..45ccdba45 100644 --- a/tests/environment-browser.test.ts +++ b/tests/environment-browser.test.ts @@ -217,7 +217,7 @@ test("installed only for a hosted operator whose handover routes the view; the b // The extension is written next to the profile, never into it. expect( await readdir(join(root ?? "", ".local/share/lazurio-browser/extension")), - ).toEqual(["background.js", "manifest.json", "webauthn.js"]); + ).toEqual(["background.js", "manifest.json", "menu.js", "webauthn.js"]); const written = await stat(join(directory, browserUnit)); // Repeated with the same entry: nothing rewritten, no reload, no restart; From 3957fa8992fcf8f2bd8488092de142b8d5edfafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= Date: Tue, 6 Oct 2026 21:15:26 +0200 Subject: [PATCH 4/4] Environment browser view: drop selection reads a navigation overtook; menu prevention in the capture phase; sorted listing (F39) Pablo's review of e225489: - A selection read that was still out when the page navigated could resolve afterwards and put the old document's text back (a copy would then take it). The hub counts the main frame's documents and drops a selection or title read that a navigation overtook. A regression test holds the read across a navigation; it fails without the check. - A page handler that stops a contextmenu event before it reaches the window bypassed the bubbling listener, and Chrome's native menu opened outside the view. The extension now prevents it in the capture phase on the window; the page's own handlers still run. Checked in Chrome for Testing 155 with a page that stops propagation: its handler ran and the default was prevented. A dispatch-order regression test covers it. - The extension listing test depended on readdir order (red on the Ubuntu CI shard); it compares the sorted list. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/decisions.md | 8 +- src/browser/people/extension.ts | 7 +- src/browser/people/hub.ts | 11 ++- tests/environment-browser-view.test.ts | 100 +++++++++++++++++++++++-- tests/environment-browser.test.ts | 4 +- 5 files changed, 115 insertions(+), 15 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index ce5de5f68..1e7c625e6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -5181,7 +5181,9 @@ F38 points 1, 5 and 6 as stated below; points 2–4 and 7 stand. service reads the selection in an isolated world when it may have changed (a released mouse button or key) and when the person presses `Ctrl` or `⌘`, just before a copy. It never enables the Runtime domain: a binding would - need it, and pages can detect it as automation during a person's sign-in. + need it, and pages can detect it as automation during a person's sign-in. A + navigation clears the selection, and a read that a navigation overtook is + dropped, so a copy never takes the previous document's text. - **Touch** is sent as touch, and a tap on a phone opens its keyboard. - The view uses the `paste` and `copy` events, so it needs no clipboard permission even inside an app's frame. @@ -5196,7 +5198,9 @@ F38 points 1, 5 and 6 as stated below; points 2–4 and 7 stand. reports no platform authenticator and no conditional mediation. Chrome's native passkey window never opens, and sites offer another way; - turns Chrome's own context menu off: it too would open outside the page. - A page's own menu still works. + The prevention runs in the capture phase on the window, so a page that + stops the event cannot let it through. The page's handlers still run, + and a page's own menu still works. - The view server keeps a DevTools session on every page: - **JavaScript dialogs** (alert, confirm, prompt, beforeunload) show in the view and are answered from it; diff --git a/src/browser/people/extension.ts b/src/browser/people/extension.ts index d0843c9c9..28b14dace 100644 --- a/src/browser/people/extension.ts +++ b/src/browser/people/extension.ts @@ -107,9 +107,10 @@ const webauthn = `// Lazurio (decision F39, root decision 0191 point 17): until const menu = `// Lazurio (decision F39 point 7): Chrome's own context menu would open // outside the page, where the people's view cannot show it. It is turned off -// for every page; a page's own menu, which it draws itself, still works. This -// listener runs last (the window, in the bubbling phase), after the page's. -window.addEventListener("contextmenu", (event) => event.preventDefault()); +// for every page: in the capture phase on the window, before any handler of +// the page can stop the event. The page's own handlers still run, so a menu +// the page draws itself still opens. +window.addEventListener("contextmenu", (event) => event.preventDefault(), true); `; export const browserExtensionFiles: Readonly> = diff --git a/src/browser/people/hub.ts b/src/browser/people/hub.ts index 57e4d54cb..f386d05a7 100644 --- a/src/browser/people/hub.ts +++ b/src/browser/people/hub.ts @@ -84,6 +84,9 @@ type Tab = { attaching: boolean; /** The view's isolated world in the main frame, until it navigates. */ world: number | null; + /** Counts the main frame's documents: a read that a navigation overtook + * belongs to the old document and is dropped. */ + document: number; /** A page opened by another, announced once its address is known. */ pendingOpen: { openerId: string; @@ -638,6 +641,7 @@ export class BrowserHub { watched: false, attaching: false, world: null, + document: 0, pendingOpen: null, selectionTimer: null, metaTimer: null, @@ -806,7 +810,10 @@ export class BrowserHub { if (tab.selectionTimer !== null) return; tab.selectionTimer = setTimeout(() => { tab.selectionTimer = null; + const document = tab.document; void this.inWorld(tab, selectionScript).then((value) => { + // A navigation overtook the read: it is the old document's text. + if (tab.document !== document) return; if (typeof value !== "string" || value === tab.selection) return; tab.selection = value; this.broadcast(tab, { t: "selection", text: value }); @@ -815,8 +822,9 @@ export class BrowserHub { } private async meta(tab: Tab): Promise { + const document = tab.document; const value = await this.inWorld(tab, metaScript); - if (typeof value !== "string") return; + if (tab.document !== document || typeof value !== "string") return; let read: { title?: unknown; icon?: unknown }; try { read = JSON.parse(value) as { title?: unknown; icon?: unknown }; @@ -993,6 +1001,7 @@ export class BrowserHub { if (frame?.parentId !== undefined) return; if (typeof frame?.id === "string") tab.mainFrameId = frame.id; tab.world = null; + tab.document += 1; // A new document has no selection: a copy must not take the old one. if (tab.selection !== "") { tab.selection = ""; diff --git a/tests/environment-browser-view.test.ts b/tests/environment-browser-view.test.ts index 38225de7b..1f0543621 100644 --- a/tests/environment-browser-view.test.ts +++ b/tests/environment-browser-view.test.ts @@ -66,6 +66,8 @@ class FakeBrowser { open = false; /** What the page's selection and title read as. */ selection = ""; + /** Holds the next selection read until it is opened (ordering tests). */ + gate: Promise | null = null; pageTitle = "Stránka z dokumentu"; constructor(initial: Target[] = []) { @@ -80,6 +82,24 @@ class FakeBrowser { return { send: async (method, params = {}, sessionId) => { if (!this.open) throw new Error("closed"); + const expression = String( + (params as { expression?: unknown }).expression, + ); + if ( + method === "Runtime.evaluate" && + expression.includes("getSelection") && + this.gate !== null + ) { + // The page's answer is what it was when the read was asked. + const answer = this.selection; + await this.gate; + this.calls.push({ + method, + params: { ...params }, + ...(sessionId === undefined ? {} : { sessionId }), + }); + return { result: { value: answer } }; + } this.calls.push({ method, params: { ...params }, @@ -628,6 +648,47 @@ test("the remote selection reaches the person for copy and cut, read when it may view.socket.close(); }); +test("a selection read that a navigation overtook is dropped: a copy never takes the previous document's text", async () => { + const browser = new FakeBrowser([tab(1)]); + const { base } = await serve(browser); + const view = await person(base, id(1)); + await until(() => view.of("info").length > 0); + browser.selection = "OLD-DOCUMENT-SECRET"; + let open = () => {}; + browser.gate = new Promise((resolve) => { + open = resolve; + }); + view.send({ + t: "mouse", + type: "mouseReleased", + x: 10, + y: 10, + button: "left", + buttons: 0, + clickCount: 1, + modifiers: 0, + }); + await until(() => + browser.calls.some((c) => c.method === "Page.createIsolatedWorld"), + ); + await settle(80); + // The page navigates while the read is still out; then the old answer + // comes back. + browser.emit( + "Page.frameNavigated", + { frame: { id: `F-S-${id(1)}`, url: "https://example.org/next" } }, + `S-${id(1)}`, + ); + await settle(30); + browser.gate = null; + open(); + await settle(150); + expect( + view.of("selection").some((m) => m.text === "OLD-DOCUMENT-SECRET"), + ).toBe(false); + view.socket.close(); +}); + // ---- Pop-ups and new tabs (F39 point 4) -------------------------------------- test("a page that can reach its opener is a pop-up; one that cannot is a new tab", async () => { @@ -966,18 +1027,41 @@ test("the extension declines passkeys and leaves other credentials alone", async ); }); -test("the extension turns Chrome's own context menu off and leaves a page's own menu working", () => { - const listeners: ((event: Event) => void)[] = []; +test("the extension turns Chrome's own context menu off even when the page stops the event, and the page's own menu still opens", () => { + // A small model of DOM dispatch on one target under the window: capture + // listeners of the window first, then the target's, then the window's + // bubbling listeners unless propagation was stopped. + const windowCapture: ((event: Event) => void)[] = []; + const windowBubble: ((event: Event) => void)[] = []; const window = { - addEventListener: (type: string, listener: (event: Event) => void) => { - if (type === "contextmenu") listeners.push(listener); + addEventListener: ( + type: string, + listener: (event: Event) => void, + capture?: boolean, + ) => { + if (type !== "contextmenu") return; + (capture === true ? windowCapture : windowBubble).push(listener); }, }; new Function("window", browserExtensionFiles["menu.js"] as string)(window); - const event = new Event("contextmenu", { cancelable: true }); - // The page's own handler ran before (a target or capture listener) and - // drew its menu; the extension only keeps Chrome's away. - for (const listener of listeners) listener(event); + let pageMenu = 0; + const pageHandler = (event: Event) => { + pageMenu += 1; + event.stopPropagation(); + }; + const event = new Event("contextmenu", { bubbles: true, cancelable: true }); + let stopped = false; + const stop = event.stopPropagation.bind(event); + event.stopPropagation = () => { + stopped = true; + stop(); + }; + for (const listener of windowCapture) listener(event); + pageHandler(event); + if (!stopped) for (const listener of windowBubble) listener(event); + expect(windowCapture).toHaveLength(1); + expect(windowBubble).toHaveLength(0); + expect(pageMenu).toBe(1); expect(event.defaultPrevented).toBe(true); }); diff --git a/tests/environment-browser.test.ts b/tests/environment-browser.test.ts index 45ccdba45..29c715d86 100644 --- a/tests/environment-browser.test.ts +++ b/tests/environment-browser.test.ts @@ -216,7 +216,9 @@ test("installed only for a hosted operator whose handover routes the view; the b ]); // The extension is written next to the profile, never into it. expect( - await readdir(join(root ?? "", ".local/share/lazurio-browser/extension")), + ( + await readdir(join(root ?? "", ".local/share/lazurio-browser/extension")) + ).sort(), ).toEqual(["background.js", "manifest.json", "menu.js", "webauthn.js"]); const written = await stat(join(directory, browserUnit));