diff --git a/.changeset/demo-capacity-sync-halt.md b/.changeset/demo-capacity-sync-halt.md new file mode 100644 index 000000000..0a625dd92 --- /dev/null +++ b/.changeset/demo-capacity-sync-halt.md @@ -0,0 +1,5 @@ +--- +'@xnetjs/runtime': minor +--- + +`NodeStoreSyncProvider` now handles hub capacity rejections gracefully: on the first `QUOTA_EXCEEDED` (over the hub's per-user cap) or `STORAGE_FULL` (hub disk full) rejection it pauses outbound sync instead of re-flooding the hub, keeps local data intact, and resumes on the next reconnect. Subscribe to the new `onSyncBlocked(listener)` API (with `SyncBlockedReason`/`SyncBlockedListener` types) to surface a "storage full" notice in your app. diff --git a/apps/web/src/lib/share-links.test.ts b/apps/web/src/lib/share-links.test.ts index 781d166ce..d085ee0e9 100644 --- a/apps/web/src/lib/share-links.test.ts +++ b/apps/web/src/lib/share-links.test.ts @@ -98,6 +98,17 @@ describe('claimShareLink', () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('nope', { status: 502 }))) await expect(claimShareLink(input, 'token')).rejects.toMatchObject({ code: 'HTTP_502' }) }) + + it('maps a network-layer failure to HUB_UNREACHABLE naming the hub (0290)', async () => { + // fetch() rejects with a bare TypeError for hub-down / CORS-less edge + // errors — the user should see an outage, not "Failed to fetch". + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new TypeError('Failed to fetch'))) + await expect(claimShareLink(input, 'token')).rejects.toMatchObject({ + name: 'ShareClaimError', + code: 'HUB_UNREACHABLE', + message: expect.stringContaining('https://hub.example.com') + }) + }) }) describe('claim error text', () => { @@ -136,6 +147,9 @@ describe('docRouteFor', () => { params: { dashboardId: 'd' } }) expect(docRouteFor('view', 'e')).toEqual({ to: '/view/$viewId', params: { viewId: 'e' } }) + expect(docRouteFor('space', 'f')).toEqual({ to: '/space/$spaceId', params: { spaceId: 'f' } }) + // Workspaces have no viewer route — a claimed bench lands home (0280/0290). + expect(docRouteFor('workspace', 'g')).toEqual({ to: '/', params: {} }) }) }) @@ -241,6 +255,13 @@ describe('hubApiFetch', () => { 'Hub request failed (500)' ) }) + + it('names the hub on network-layer failures instead of "Failed to fetch" (0290)', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new TypeError('Failed to fetch'))) + await expect(hubApiFetch('https://hub.x', 'tok', '/shares/links')).rejects.toThrow( + "Your hub (https://hub.x) isn't responding" + ) + }) }) describe('isPrivateHubHost and URL normalization', () => { diff --git a/apps/web/src/lib/share-links.ts b/apps/web/src/lib/share-links.ts index f3ff9a118..c56030bc6 100644 --- a/apps/web/src/lib/share-links.ts +++ b/apps/web/src/lib/share-links.ts @@ -15,7 +15,7 @@ export type ShareLinkInput = { export type ShareClaimResult = { resource: string - docType: 'page' | 'database' | 'canvas' | 'dashboard' | 'view' | 'space' + docType: 'page' | 'database' | 'canvas' | 'dashboard' | 'view' | 'space' | 'workspace' role: 'read' | 'comment' | 'write' endpoint: string } @@ -88,15 +88,25 @@ export async function claimShareLink( authToken: string ): Promise { const hub = normalizeHubHttpUrl(input.hub) - const response = await fetch(`${hub}/shares/links/${encodeURIComponent(input.linkId)}/claim`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${authToken}` - }, - body: JSON.stringify({ secret: input.secret }), - cache: 'no-store' - }) + let response: Response + try { + response = await fetch(`${hub}/shares/links/${encodeURIComponent(input.linkId)}/claim`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${authToken}` + }, + body: JSON.stringify({ secret: input.secret }), + cache: 'no-store' + }) + } catch { + // Network-layer failure (hub down / edge error without CORS headers) — + // surface the hub, not a bare "Failed to fetch" (exploration 0290). + throw new ShareClaimError( + 'HUB_UNREACHABLE', + `The hub issuing this link (${hub}) isn't responding — it may be down or restarting. Try again shortly.` + ) + } const data = (await response.json().catch(() => null)) as | (ShareClaimResult & { code?: string; error?: string }) @@ -129,6 +139,8 @@ export function shareClaimErrorMessage(code: string): string { return 'This share link is missing or has a corrupted secret. Copy the full link and try again.' case 'RATE_LIMITED': return 'Too many attempts. Wait a minute and try again.' + case 'HUB_UNREACHABLE': + return "The hub issuing this link isn't responding — it may be down or restarting. Try again shortly." default: return 'The share link could not be claimed.' } @@ -218,15 +230,26 @@ export async function hubApiFetch( path: string, init: { method?: string; body?: unknown } = {} ): Promise { - const response = await fetch(`${hubHttpUrl}${path}`, { - method: init.method ?? 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${authToken}` - }, - ...(init.body !== undefined ? { body: JSON.stringify(init.body) } : {}), - cache: 'no-store' - }) + let response: Response + try { + response = await fetch(`${hubHttpUrl}${path}`, { + method: init.method ?? 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${authToken}` + }, + ...(init.body !== undefined ? { body: JSON.stringify(init.body) } : {}), + cache: 'no-store' + }) + } catch { + // fetch() rejects with a bare TypeError ("Failed to fetch") for every + // network-layer failure — including an edge 502 served without CORS + // headers while the hub is down (exploration 0290). Name the hub so the + // user sees an outage, not a mystery. + throw new Error( + `Your hub (${hubHttpUrl}) isn't responding — it may be down or restarting. Try again shortly.` + ) + } const data = (await response.json().catch(() => null)) as { error?: string } | null if (!response.ok) { throw new Error(data?.error ?? `Hub request failed (${response.status})`) @@ -250,6 +273,10 @@ export function docRouteFor( return { to: '/view/$viewId', params: { viewId: resource } } case 'space': return { to: '/space/$spaceId', params: { spaceId: resource } } + case 'workspace': + // Workspaces have no viewer route; land home — the granted node syncs + // and appears in the receiver's workspace switcher (0280). + return { to: '/', params: {} } default: return { to: '/doc/$docId', params: { docId: resource } } } diff --git a/docs/explorations/0290_[_]_SHARE_LINK_GENERATION_FAILURE_MODES.md b/docs/explorations/0290_[_]_SHARE_LINK_GENERATION_FAILURE_MODES.md new file mode 100644 index 000000000..e535cab30 --- /dev/null +++ b/docs/explorations/0290_[_]_SHARE_LINK_GENERATION_FAILURE_MODES.md @@ -0,0 +1,463 @@ +# Share-Link Generation: Failure Modes And Fixes + +## Problem Statement + +A user reports that **generating a share link fails** — e.g. "generating a +link to share a page." This exploration reproduces the share-link flow +locally against a real hub, isolates what actually fails (and what does +_not_), and recommends concrete fixes. + +Bottom line up front: **sharing a _page_ works** end-to-end against a +reachable hub — I generated, claimed, and resolved a page link locally with +no errors. The real failures are elsewhere: + +0. **⭐ The actual production report ("Failed to fetch" on `xnet.fyi/app` → + `hub.xnet.fyi`) is a hub _outage_, not a share-link bug.** `hub.xnet.fyi` + currently returns **`HTTP 502 "Application failed to respond"`** from the + Railway edge for _every_ path (including `/health`). That 502 comes from + the edge, **before** the hub's `cors()` runs, so it carries **no + `Access-Control-Allow-Origin` header** — which is precisely what turns a + CORS-preflighted `POST /shares/links` into a browser `TypeError: Failed to + fetch`. Fix is operational (restart/redeploy the hub), not code. See + [Production Outage](#production-outage-hubxnetfyi-returns-502--the-failed-to-fetch-report). +1. **Sharing a _workspace/bench_ is broken** — the client sends + `docType: 'workspace'`, which the hub rejects with `400 INVALID_BODY` + ("Missing docId, docType, or role"). Definite bug. +2. **Local-first default = no hub** — the app boots with no hub, and the + Share dialog then only says _"Connect to a hub to create share links."_ + with no way to act. This is what most "sharing is broken" reports are. +3. **Private / `localhost` hubs mint links nobody else can open** — + generation "succeeds" but produces a `http://localhost:4444/s/…` URL that + only resolves on the issuing machine. + +## Executive Summary + +| Scenario | Result | Root cause | +| --- | --- | --- | +| **`xnet.fyi/app` → `hub.xnet.fyi` (the reported case)** | ❌ **`Failed to fetch`** | **Hub is down — Railway edge returns 502 with no CORS headers** | +| Share a **page** (hub connected, you own the doc) | ✅ works | — | +| Share a **database / canvas / dashboard / view / space** | ✅ works | — | +| Share a **workspace / bench** | ❌ `400 INVALID_BODY` | Hub `SHARE_DOC_TYPES` omits `'workspace'` | +| Share **anything with no hub connected** | ⚠️ blocked, no CTA | Local-first default; dialog dead-ends | +| Share from a **`localhost`/LAN hub** | ⚠️ link unusable off-machine | Private hub host in the URL | +| Share a doc **owned by another DID** (search-index recorded owner) | ❌ `403 FORBIDDEN` | `canManageShares` owner check | + +I verified the production row and rows 2–6 directly (curl against +`hub.xnet.fyi`; Playwright against the running app; Node probes against a +live hub). The last row is confirmed by code inspection. + +## Production Outage: hub.xnet.fyi Returns 502 (the `Failed to fetch` report) + +The originally reported symptom — the link-generation modal on the hosted +demo app (`xnet.fyi/app`, connected to `hub.xnet.fyi`) failing with **"Failed +to fetch"** — reproduces from a plain `curl`, and it is **not** a share-link +code path at all. The hub is simply **down**. + +Probed live on 2026-07-10: + +``` +$ curl -v https://hub.xnet.fyi/health +* Trying 69.46.46.121:443... # DNS → Railway (xjthmykc.up.railway.app) +* SSL certificate verify ok. # TLS handshake fine, edge is up +< HTTP/2 502 +< content-type: application/json +< server: railway-hikari +< x-railway-fallback: true # ← Railway edge fallback, app did NOT respond +< x-railway-edge: sjc1 +{"status":"error","code":502,"message":"Application failed to respond","request_id":"…"} +``` + +Every path (including `/health` and `/shares/links`) returns the same 502. +For comparison, `xnet.fyi` and `cloud-staging.xnet.fyi` respond normally from +the same network, so this is `hub.xnet.fyi` specifically — the Railway +container is crashed / OOM / not booting, and the edge is serving its +fallback error page. + +### Why a hub outage surfaces as `Failed to fetch`, not a clean error + +```mermaid +sequenceDiagram + participant B as Browser (xnet.fyi/app) + participant E as Railway edge + participant H as Hub app (cors() lives here) + + Note over B: POST /shares/links carries Authorization + Content-Type
→ CORS-preflighted (OPTIONS first) + B->>E: OPTIONS/POST /shares/links (Origin: https://xnet.fyi) + E->>H: forward upstream + H--xE: no response (crashed / OOM / not booted) + E-->>B: 502 "Application failed to respond"
❌ NO Access-Control-Allow-Origin + Note over B: Response lacks CORS headers →
browser blocks it → fetch() throws
TypeError: "Failed to fetch" + Note over B: hubApiFetch rethrows verbatim →
ShareDialog shows "Failed to fetch" +``` + +The crucial subtlety: the hub's `app.use('*', cors())` +(`packages/hub/src/server.ts:150`, added in PR #398) can only attach +`Access-Control-Allow-Origin` **when the request reaches the hub process**. +A Railway edge 502 never gets there, so the error page has no CORS headers. +A CORS-preflighted request whose response lacks those headers is rejected by +the browser as a network `TypeError` — surfaced as the generic **"Failed to +fetch"**, indistinguishable (to the user) from a real CORS misconfiguration. + +`hubApiFetch` (`apps/web/src/lib/share-links.ts:215`) does not catch the +network `TypeError`, so `createLink` → `handleCreate` +(`apps/web/src/components/ShareDialog.tsx:170`) sets `actionError = +"Failed to fetch"` verbatim. The same outage also breaks the `wss://hub.xnet.fyi` +sync socket, so the app is running purely local-first and the modal's +list-links `GET`s fail too (the user just notices the click that errors). + +This is the exact scenario captured in the team memory note +_"Share dialog CORS + hub outage (#398): … Railway 502 = USER restart."_ — +the immediate remedy is to **restart / redeploy the Railway hub service**, +then confirm `GET https://hub.xnet.fyi/health` returns `200`. + +**Why the hub crashed (likely root cause):** a restart alone may not hold. The +demo hub's per-user quota and daily eviction are **not enforced**, so one +active user filled the 500 MB Railway volume with >1 GB of `node_changes` data +— and a full SQLite volume crashes the hub on the next write/boot, producing +exactly this 502. Restart, but also **truncate the demo volume** and fix the +guardrails. Full analysis and fixes: +[0291_[_]_DEMO_HUB_RUNAWAY_STORAGE…](0291_%5B_%5D_DEMO_HUB_RUNAWAY_STORAGE_QUOTA_AND_EVICTION_NOT_ENFORCED.md). + +### What this changes about the diagnosis + +The `Failed to fetch` report is an **operational outage**, not the +share-link code. But it exposes two worthwhile product gaps: + +- **The app can't tell "hub down" from "CORS bug" from "offline".** All three + collapse into "Failed to fetch". `hubApiFetch` should catch the network + `TypeError` and, when the hub is otherwise known (host reachable, socket + down), surface something like _"Your hub (hub.xnet.fyi) isn't responding — + it may be restarting. Try again shortly."_ instead of the raw string. +- **There is no uptime signal for `hub.xnet.fyi`.** A 502 on the demo hub + silently breaks sharing, sync, forms, and files for every hosted user. + A health-check monitor + auto-restart would have caught this before a user + did. + +## Reproduction Environment + +- Hub: `node --import tsx packages/hub/src/cli.ts --port 4444 --storage memory` + (defaults to UCAN auth on). +- Web app: `vite` dev server, driven with Playwright. +- Onboarding passkey: bypassed the WebAuthn wall with a CDP **virtual + authenticator** (`WebAuthn.addVirtualAuthenticator`). The PRF extension + is unsupported by the virtual authenticator, so the app's non-PRF + **fallback** path minted a real `did:key:z6Mk…` identity — faithful to a + real user, not the `xnet:test:bypass` shortcut. +- Hub connection: `localStorage['xnet:hub-url'] = 'ws://localhost:4444'`, + then reload; status bar went to **"synced"**. + +Creating a page (`/doc/g9wyvf2yzv`) → **Share** → **New link** produced: + +``` +http://localhost:4444/s/fMZI_06VwjOH#s=iXJlQVaqKdhVh9efgUMdaJUE5aS9JvVr +``` + +Network: `POST /shares/links → 200`. So the page path is healthy. + +A Node probe minting a UCAN (mirroring `useHubAuthToken`) and hitting the +endpoint across every `docType`: + +``` +docType=page -> 200 +docType=database -> 200 +docType=canvas -> 200 +docType=dashboard -> 200 +docType=view -> 200 +docType=space -> 200 +docType=workspace -> 400 {"code":"INVALID_BODY","error":"Missing docId, docType, or role"} +``` + +And a full round-trip (owner A creates → recipient B claims → interstitial): + +``` +create: http://localhost:4444/s/ziWlD0bxNrfc#s=… +claim status: 200 {"resource":"roundtrip_page","docType":"page","role":"write",…} +interstitial GET /s/:id status: 200 content-type: text/html +``` + +## Current State In The Repository + +### Generation call chain + +``` +ShareDialog.handleCreate apps/web/src/components/ShareDialog.tsx:158 + └─ useShareLinks.createLink apps/web/src/hooks/useShareLinks.ts:190 + └─ hubApiFetch POST /shares/links apps/web/src/lib/share-links.ts:215 + └─ hub POST /links packages/hub/src/routes/share-links.ts:142 + ├─ requireAuth (UCAN) packages/hub/src/server.ts:432 + ├─ isShareDocType(docType) packages/hub/src/routes/share-links.ts:156 + └─ canManageShares(did, docId) packages/hub/src/routes/share-links.ts:91 +``` + +The `docType` mapping in the workbench header: +`packages/…`→ `apps/web/src/workbench/EditorHeader.tsx:29` maps node types to +`ShareDocType`. `page → 'page'` (valid), so the page header Share button is +correct. + +### The `workspace` mismatch (bug #1) + +- Client union **includes** `'workspace'`: + `apps/web/src/hooks/useShareLinks.ts:13-22`. +- `WorkspaceSwitcher` sends it: `apps/web/src/workbench/WorkspaceSwitcher.tsx:215` + (`docType="workspace"`). +- Hub **rejects** it — `SHARE_DOC_TYPES` has no `'workspace'`: + `packages/hub/src/routes/share-links.ts:35`, validated at `:156`. +- The claim side is also unwired for it: `ShareClaimResult['docType']` union + omits `'workspace'` (`apps/web/src/lib/share-links.ts:17`) and + `docRouteFor` has no `'workspace'` case, falling through to `/doc/$docId` + (`apps/web/src/lib/share-links.ts:238`). + +So workspace sharing is half-implemented (added in exploration 0280 on the +client, never completed on the hub or the claim/route side). + +### The no-hub dead-end (bug #2) + +- Default boot logs `hub: (none — local-first…)` + (`apps/web/src/boot/use-boot-sequence.ts:37`); hub is resolved from + `localStorage['xnet:hub-url']` / `VITE_HUB_URL` / `?hub=` + (`apps/web/src/lib/hub-url.ts`). +- With no hub, `useShareLinks` is not `ready` + (`apps/web/src/hooks/useShareLinks.ts:109`) and the dialog renders only + _"Connect to a hub to create share links."_ + (`apps/web/src/components/ShareDialog.tsx:238-240`) — **no button to + connect one**. A hub _can_ be connected from the status-bar chip + (`title="Hub: disconnected · local-ready"`, opens a dialog), but the Share + dialog never points there. +- If a token is somehow empty, `hubApiFetch` still fires `Authorization: + Bearer ` and the hub returns `401 UNAUTHORIZED` + (`packages/hub/src/auth/ucan.ts`). + +### The private-hub link (bug #3) + +- `isPrivateHubHost` (`apps/web/src/lib/share-links.ts:268`) already detects + `localhost`/RFC-1918/`.local`; the dialog shows an amber warning + (`apps/web/src/components/ShareDialog.tsx:242-249`). But it still generates + a `http://localhost:…` link — copyable and share-looking, yet dead for any + recipient. The hub builds that URL from `publicUrl ?? ws://localhost:port` + (`packages/hub/src/routes/share-links.ts:65-68,188-191`). + +### Ownership gate (edge, bug #4) + +`canManageShares` (`packages/hub/src/routes/share-links.ts:91-101`): if a +`docMeta` row exists with a different `ownerDid` and the caller has no +`admin` grant → `403 FORBIDDEN`. `ownerDid` is only ever recorded via the +**search-index** WS path (`packages/hub/src/ws/handlers/search-index.ts:40` +→ `packages/hub/src/services/query.ts:106`), keyed to the DID that sent the +index update. So a user who **rotated their identity** (recovery on a new +device, new `did:key`) can be locked out of sharing their own previously +indexed doc. + +Conversely — and worth flagging — if **no** `docMeta` exists (the common +case; plain page/CRDT sync never writes it), `canManageShares` returns +`true`, so _any_ authenticated DID can mint links for _any_ `docId`. + +## External Research + +- **WebAuthn PRF + virtual authenticators.** Chrome's CDP + `WebAuthn.addVirtualAuthenticator` does not advertise the `hmac-secret` + extension that WebAuthn PRF relies on, so PRF-derived-key flows must have a + fallback — which xNet has (`packages/identity/src/passkey/create.ts`, + `support.ts`). This is the standard pattern (see Chromium + `VirtualAuthenticatorOptions`; MDN "Web Authentication extensions → prf"). +- **Fragment secrets.** Carrying the capability secret in the URL `#fragment` + (never sent to the server) is the same technique used by password managers + and E2E "secret link" tools (e.g. 1Password sharing, Firefox Send's model). + xNet's hub only stores `sha256(secret)` and returns the secret exactly once + (`packages/hub/src/routes/share-links.ts:47,170-191`) — correct. +- **Prior fix in this codebase.** PR #398 added global `app.use('*', cors())` + (`packages/hub/src/server.ts:150`) so authenticated cross-origin POSTs + survive preflight. I confirmed the preflight is answered + (`access-control-allow-origin: *`) **when the request reaches the hub**. + But that guarantee evaporates during an outage: a **Railway edge 502** + (`x-railway-fallback: true`) is served _before_ the hub runs, with no CORS + headers — reintroducing the same browser-visible "Failed to fetch". The + application-layer CORS fix cannot cover edge/proxy error pages; only hub + uptime (or edge-injected CORS headers) can. This is the production case + observed here. + +## Key Findings + +```mermaid +flowchart TD + A[Click 'New link'] --> B{Hub connected?} + B -- no --> B1[Dialog: 'Connect to a hub…'
no CTA — dead end]:::warn + B -- yes --> C{Valid UCAN token?} + C -- no/empty --> C1[401 UNAUTHORIZED]:::bad + C -- yes --> D{docType in hub allow-list?} + D -- workspace --> D1[400 INVALID_BODY
'Missing docId, docType, or role']:::bad + D -- page/db/canvas/dashboard/view/space --> E{canManageShares?} + E -- other DID owns it --> E1[403 FORBIDDEN]:::bad + E -- ok / no owner --> F[200 — link minted]:::good + F --> G{Hub publicly reachable?} + G -- localhost/LAN --> G1[Link works only on this machine]:::warn + G -- public --> G2[Shareable link]:::good + + classDef bad fill:#fdd,stroke:#c00; + classDef warn fill:#ffe9c7,stroke:#e69500; + classDef good fill:#dfd,stroke:#0a0; +``` + +1. **Page sharing is not broken** with a reachable hub and matching identity. +2. **Workspace sharing is broken** at the protocol layer (`docType` allow-list + drift between client and hub). +3. The **no-hub** experience is the most common "sharing is broken" report: + local-first is the default and the Share dialog dead-ends. +4. **Private-hub** links are generated but non-functional off-machine. +5. Two ownership edges: a rotated identity locks you out (`403`); a missing + `docMeta` lets anyone mint links (over-permissive). + +## Options And Tradeoffs + +### Bug #1 — `workspace` docType + +| Option | What | Tradeoff | +| --- | --- | --- | +| **A. Add `'workspace'` to the hub allow-list** (recommended) | Add to `SHARE_DOC_TYPES` (`share-links.ts:35`), the claim `ShareClaimResult['docType']` union, and a `docRouteFor` case. | Small, closes the drift; must also confirm the claim → route → sync path resolves a workspace node. Protocol-surface change → **minor** changeset for `@xnetjs/hub`? No: the hub isn't a publishable lib boundary here, but the accepted-values change is consumer-visible — bump per the diff. | +| **B. Remove `'workspace'` from the client** | Drop the union member + the `WorkspaceSwitcher` Share entry until the hub supports it. | Fastest to stop the error, but removes a feature 0280 intended to ship. | +| **C. Generic "shareable node" type** | Collapse doc-type validation to "is this a shareable node id?" and carry type as advisory. | Bigger refactor; loses the type-scoped routing/role semantics (esp. `space` subtree grants). | + +### Bug #2 — no-hub dead-end + +| Option | What | Tradeoff | +| --- | --- | --- | +| **A. In-dialog "Connect a hub" CTA** (recommended) | When `!ready`, render a button that opens the existing hub-connection dialog (the status-bar chip target). | Small UX add; turns a dead-end into a path. | +| **B. Ship a default hub** | Point `VITE_HUB_URL` at a managed hub for hosted builds. | Changes the local-first promise; only for the hosted app, gated on consent. | +| **C. Explain-only** | Improve copy ("Sharing needs a hub because links are claimed server-side…"). | Cheap, but still no action. | + +### Bug #3 — private-hub links + +| Option | What | Tradeoff | +| --- | --- | --- | +| **A. Escalate the warning + gate copy** (recommended) | Keep generating (LAN sharing is legitimate) but label the URL "Local only" and require a confirm to copy/QR. | Preserves LAN use; prevents "why doesn't my link work" confusion. | +| **B. Block generation on private hubs** | Refuse to mint when `isPrivateHubHost`. | Breaks legitimate same-LAN/in-person QR handoff. | + +### Bug #4 — ownership edges + +| Option | What | Tradeoff | +| --- | --- | --- | +| **A. Record `ownerDid` on first node write, not just index** | Have node-relay stamp `docMeta.ownerDid` when a doc is first seen. | Closes the "anyone can mint links for any docId" hole; must handle legacy docs and multi-writer docs carefully. | +| **B. Better `403` copy + self-heal for identity rotation** | Detect owner-mismatch and offer recovery-phrase re-link / admin-grant path. | Narrow; doesn't fix the over-permissive default. | + +## Recommendation + +**First, resolve the outage (this is what the user actually hit):** + +0. **Restart / redeploy the `hub.xnet.fyi` Railway service now**, then verify + `curl https://hub.xnet.fyi/health` → `200`. Check the Railway logs for the + crash cause (OOM, boot failure, Litestream/VACUUM — cf. explorations 0258 + Cloud HA and the cold-open-stall note). Then add uptime monitoring + + auto-restart so a hub 502 pages an operator, not a user. + +**Then ship the two high-signal, low-risk code fixes, then the hardening:** + +1. **Fix `workspace` (Bug #1, Option A)** — align the hub allow-list, the + claim union, and `docRouteFor`, and add a hub test asserting all seven + `ShareDocType` values round-trip. This is a true generation failure with a + confusing error string; it's the clearest "share fails" defect. +2. **Add the in-dialog "Connect a hub" CTA (Bug #2, Option A)** — this is what + most local-first users actually hit. Reuse the status-bar hub dialog. +3. **Label private-hub links "Local only" with a copy-confirm (Bug #3, + Option A).** +4. **Follow up on ownership (Bug #4)** — separately, decide between stamping + `ownerDid` on first write vs. keeping the permissive default; today's + behaviour is a latent authz smell, not the user's immediate bug. + +## Example Code + +Bug #1 — hub allow-list (`packages/hub/src/routes/share-links.ts:35`): + +```ts +// Add 'workspace' (saved shell layouts / benches — exploration 0280). +const SHARE_DOC_TYPES = [ + 'page', 'database', 'canvas', 'dashboard', 'view', 'space', 'workspace' +] as const +``` + +Claim union + route (`apps/web/src/lib/share-links.ts:17,238`): + +```ts +export type ShareClaimResult = { + resource: string + docType: 'page' | 'database' | 'canvas' | 'dashboard' | 'view' | 'space' | 'workspace' + role: 'read' | 'comment' | 'write' + endpoint: string +} + +// docRouteFor(): add +case 'workspace': + return { to: '/workspace/$workspaceId', params: { workspaceId: resource } } +``` + +Bug #2 — Share dialog CTA (`apps/web/src/components/ShareDialog.tsx:238`): + +```tsx +{!ready && tab !== 'permissions' && ( +
+

Share links are claimed on a hub — connect one to create links.

+ +
+)} +``` + +## Risks And Open Questions + +- **Does the claim → route → sync path actually resolve a workspace node?** + Adding `'workspace'` to the allow-list only fixes generation; exploration + 0280's `xnet:Workspace` node must sync and open at `/workspace/$id` for the + recipient. Needs an end-to-end test, not just a 200 on `POST /links`. +- **Over-permissive `canManageShares`.** With no `docMeta`, any authenticated + DID can mint links for any `docId`. If we stamp `ownerDid` on first write, + we must not lock out legitimate multi-writer/collaborative docs or legacy + data. +- **Identity rotation → 403.** Recovery on a new device yields a new + `did:key`; previously indexed docs then reject sharing. Is there a re-link + path? (Ties into the account-recovery work, 0243.) +- **Changeset bump.** Changing the hub's accepted `docType` set and the + client `ShareClaimResult` union is a wire/contract change — bump from the + diff (per CLAUDE.md), likely **minor** for the affected publishable + packages. + +## Implementation Checklist + +- [ ] **Restart/redeploy `hub.xnet.fyi` and confirm `/health` → 200** (unblocks the reported "Failed to fetch"). +- [ ] Investigate the Railway crash cause from logs; add a health-check monitor + alerting/auto-restart for the demo hub. +- [x] Make `hubApiFetch` catch the network `TypeError` and surface a "hub unreachable / may be restarting" message instead of raw "Failed to fetch" (`apps/web/src/lib/share-links.ts:215`). +- [x] Add `'workspace'` to `SHARE_DOC_TYPES` in `packages/hub/src/routes/share-links.ts`. +- [x] Extend `ShareClaimResult['docType']` union in `apps/web/src/lib/share-links.ts`. +- [x] Add a `'workspace'` case to `docRouteFor` (route target for a claimed bench). +- [ ] Verify the recipient's claim opens the workspace node (sync + route) end-to-end. +- [ ] Add an in-dialog "Connect a hub" CTA to `ShareDialog` when `!ready`, wired to the existing hub-connection dialog. +- [ ] Label private-hub links "Local only" and add a copy/QR confirm (keep LAN sharing). +- [ ] (Follow-up) Decide `ownerDid`-on-first-write vs. permissive default; write a hub test for the chosen behaviour. +- [x] Add a hub test asserting every `ShareDocType` value returns 200 from `POST /shares/links`. +- [x] Write the changeset(s) reflecting the `docType`/union contract change. + +## Validation Checklist + +- [ ] `curl https://hub.xnet.fyi/health` returns **200** (not 502); a browser `POST /shares/links` from `xnet.fyi/app` succeeds. +- [ ] With the hub deliberately stopped, the Share dialog shows a "hub unreachable" message rather than raw "Failed to fetch". +- [x] Node probe: `POST /shares/links` returns **200** for all of + page/database/canvas/dashboard/view/space/**workspace**. +- [ ] Browser: with a hub connected, **New link** for a page, a database, and + a **bench** each yields a copyable URL (no error banner). +- [ ] Browser: with **no** hub, the Share dialog shows a working **Connect a + hub** button that lands you connected, after which **New link** works. +- [ ] Round-trip: a second identity **claims** a workspace link and lands on + the workspace view. +- [ ] Private hub: link is labelled "Local only"; copy requires confirm. +- [ ] Ownership: a non-owner (per `docMeta`) gets a clear `403` message, and + the chosen owner-stamping behaviour matches the new hub test. + +## References + +- `apps/web/src/components/ShareDialog.tsx` — the dialog + `handleCreate`. +- `apps/web/src/hooks/useShareLinks.ts` — `createLink`, `ShareDocType`, hub API. +- `apps/web/src/lib/share-links.ts` — `hubApiFetch`, claim/parse, `isPrivateHubHost`, `docRouteFor`. +- `apps/web/src/workbench/EditorHeader.tsx` / `WorkspaceSwitcher.tsx` — Share entry points + docType mapping. +- `packages/hub/src/routes/share-links.ts` — `POST /links`, `SHARE_DOC_TYPES`, `canManageShares`, secret hashing. +- `packages/hub/src/server.ts` — route mount + global `cors()` (PR #398). +- `packages/hub/src/services/query.ts`, `ws/handlers/search-index.ts` — where `docMeta.ownerDid` is recorded. +- `apps/web/src/lib/hub-url.ts`, `apps/web/src/boot/use-boot-sequence.ts` — hub resolution + local-first default. +- `packages/hub/test/share-links.test.ts` — existing generation/claim/CORS tests (extend here). +- Related explorations: 0169 (durable share links), 0179 (Spaces unified sharing), 0280 (malleable workbench / `xnet:Workspace`), 0243 (account recovery / identity rotation). diff --git a/docs/explorations/0291_[_]_DEMO_HUB_RUNAWAY_STORAGE_QUOTA_AND_EVICTION_NOT_ENFORCED.md b/docs/explorations/0291_[_]_DEMO_HUB_RUNAWAY_STORAGE_QUOTA_AND_EVICTION_NOT_ENFORCED.md new file mode 100644 index 000000000..486788524 --- /dev/null +++ b/docs/explorations/0291_[_]_DEMO_HUB_RUNAWAY_STORAGE_QUOTA_AND_EVICTION_NOT_ENFORCED.md @@ -0,0 +1,334 @@ +# Demo Hub Runaway Storage: Quota And Eviction Are Not Enforced + +## Problem Statement + +The `hub.xnet.fyi` demo server (Railway, 500 MB disk) is supposed to keep +itself small with two guardrails: + +1. a **10 MB per-user storage cap**, and +2. a **daily clear-out** of all demo data. + +Neither is happening. A single active user has accumulated **>1 GB**, which +overruns the 500 MB Railway volume — and (see exploration 0290) a full volume +is the most likely reason the hub is now returning `502 "Application failed to +respond"`, which surfaces in the app as _"Failed to fetch"_ when generating a +share link. This exploration traces why both guardrails are inert and how to +fix them. + +## Executive Summary + +Two independent failures, both verified against the code and the running +deployment: + +- **The 10 MB demo quota is advisory-only.** `demoOverrides.quota` (10 MB) is + computed and sent to clients in the WebSocket handshake as a _hint_, but the + server-side enforcers (Backup/File services) are wired to `defaultQuota` + (**1 GB**), and the **primary growth path — the append-only `node_changes` + CRDT log — has no per-user quota check at all.** So sync data grows without + limit. +- **Daily eviction is dead code.** `EvictionService` exists, is exported, and + is unit-tested against a **mock** store — but it is **never instantiated or + started** in the hub, **no real storage backend implements + `EvictionStorage`**, and `.touch()` is never called. It has never run. And + even its design (evict users _inactive >24 h_) would never clear an + **active** daily user, so it isn't a "daily clear" in the first place. +- **The Railway volume is persistent** (`RAILWAY_VOLUME_MOUNT_PATH`), so data + also never clears on restart or redeploy. + +Net: `--demo` is on, but demo mode enforces nothing. One active user fills the +disk; the hub eventually crashes on a full volume. + +```mermaid +flowchart LR + U[Active user syncs nodes/docs] -->|node-relay| A[appendNodeChange] + A -->|INSERT OR IGNORE, no quota| T[(node_changes
append-only log)] + A -.->|no .touch(did)| X[last_active table
❌ does not exist] + T --> G[Unbounded growth >1 GB] + G --> F[500 MB Railway volume FULL] + F --> C[SQLite write / boot failure] + C --> E[Hub crash → Railway 502] + subgraph guardrails that should stop this + Q[10 MB per-user quota]:::dead + V[Daily eviction sweep]:::dead + end + Q -. advisory only / wired to 1 GB .-> A + V -. never instantiated, no storage .-> T + classDef dead fill:#fdd,stroke:#c00; +``` + +## Current State In The Repository + +### Demo mode is genuinely enabled + +- Railway start command passes `--demo` + (`railway.toml` → `startCommand`), so `config.demo === true`. +- `--demo` parsed at `packages/hub/src/cli.ts:77,107`; `demoOverrides` + resolved at `packages/hub/src/config.ts:139-140` via `getDemoOverrides` + (`config.ts:84-95`). +- Demo values (`packages/hub/src/types.ts:119-138`): + + ```ts + export const DEMO_DEFAULTS: DemoOverrides = { + quota: 10 * 1024 * 1024, // 10 MB per-user + maxDocs: 50, + maxBlob: 2 * 1024 * 1024, // 2 MB + evictionTtl: 24 * 60 * 60 * 1000, // 24 h inactivity + evictionInterval: 60 * 60 * 1000 // hourly sweep + } + ``` + +So the config is correct. Nothing consumes it as an enforcer. + +### Failure A — the 10 MB quota never reaches an enforcer + +- **Backup/File services are wired to `defaultQuota` (1 GB), not the demo + quota** (`packages/hub/src/server.ts:170-178`): + + ```ts + const backup = new BackupService(storage, { + maxQuotaBytes: config.defaultQuota, // 1 GB — should be demoOverrides.quota + maxBlobSize: config.maxBlobSize // 50 MB — should be demoOverrides.maxBlob + }) + const files = new FileService(storage, { maxStoragePerUser: config.defaultQuota }) + ``` + + The enforcement _logic_ is fine (`services/backup.ts:34-46` → + `QUOTA_EXCEEDED`/`BLOB_TOO_LARGE`; `services/files.ts:56-57`) — it's just + checking against 1 GB. + +- **The append-only `node_changes` log has no quota check at all.** The + ingestion path `packages/hub/src/services/node-relay.ts:129-145` validates + hash/DID/signature/mentions and then calls `storage.appendNodeChange(...)` + unconditionally. `author_did` is stored on every row + (`storage/sqlite.ts:320`) but never summed or capped. `appendNodeChange` + (`sqlite.ts:1141-1148,2089`) is `INSERT OR IGNORE` with no size/count gate. + This is the >1 GB grower. The only deletion path is `clearNodeChanges(room)` + (`sqlite.ts:1163-1165`), reachable only via an explicit `node-clear` request + (`node-relay.ts:154-160`) — never automatic. + +- **`demoOverrides.quota` is used in exactly two informational places:** the + WS handshake `demoLimits.quotaBytes` sent to clients (`server.ts:797-803`, + advisory — the client is trusted to self-limit) and a startup `console.log` + (`cli.ts:130`). + +- Other unbounded per-user stores with no quota gate: `doc_state` (Yjs blobs, + `sqlite.ts:51,843`, persisted from `pool/node-pool.ts:97-109`), `doc_meta`, + etc. Only `backups`/`file_meta` have quota logic (set to 1 GB). + +### Failure B — eviction is unwired dead code + +`EvictionService` (`packages/hub/src/services/eviction.ts`) is fully written — +`start()` does an immediate sweep + `setInterval` (`:34-46`), `evict()` +deletes inactive users (`:62-84`), `touch()` records activity (`:57-59`). But: + +- **Never instantiated in production.** `grep "new EvictionService"` across + `packages/hub/src` → zero hits (only `test/eviction.test.ts`). The lifecycle + start block (`server.ts:671-689`) starts telemetry/awareness/discovery/ + federation/crawl — **not** eviction. `.touch()` is never called on any + request path, so `last_active` is never recorded. +- **No storage backend implements `EvictionStorage`.** The interface needs + `upsertActivity`/`getInactiveDids`/`deleteUserData`/`deleteActivity` + (`eviction.ts:12-21`); none of these (nor a `last_active`/`user_activity` + table) exist in `storage/sqlite.ts`, `storage/memory.ts`, or + `storage/interface.ts`. So the service cannot function even if started. +- **Wrong semantics for "daily clear."** `evict()` deletes DIDs with + `last_active < now - evictionTtl` (24 h). An **active** user (the one user, + using it daily) is never idle for 24 h → never evicted. Inactivity eviction + is not the same as a daily wipe. +- **Green tests hide it.** `test/eviction.test.ts` tests the service against a + **mock** `EvictionStorage` (`:10-38`) and asserts `DEMO_DEFAULTS` + (`:190-198`). Nothing asserts wiring into the server or a real storage impl. + Classic "unit-tested in isolation, never integrated." + +### The volume is persistent (restart won't help) + +`config.ts:107-111` resolves `dataDir = RAILWAY_VOLUME_MOUNT_PATH ?? +HUB_DATA_DIR ?? cliOptions.dataDir`. Railway mounts a **persistent** volume, +so `--data /data` is overridden and data survives restarts/redeploys. Even a +hypothetical "clear on boot" would not empty the volume. The Railway +`startCommand` also bypasses the Litestream `CMD` entrypoint (which itself +builds the command _without_ `--demo`, a separate latent inconsistency in +`packages/hub/litestream-entrypoint.sh:13`). + +### Likely link to the 502 outage (0290) + +`/health`'s `usedBytes` (`server.ts:383`, `data-usage.ts:34-63`) is a +`stat`-sum for display and drives no enforcement. With >1 GB of `node_changes` +against a 500 MB volume, SQLite writes fail with `SQLITE_FULL`; a +schema/migration write or WAL checkpoint on boot then throws, the process +crashes, the Railway healthcheck fails, and the edge serves `502`. That is the +most probable root of exploration 0290's outage — the runaway data and the +"Failed to fetch" are one incident. + +## External Research + +- **Multi-tenant quota patterns.** Durable per-tenant caps are enforced at the + _write path_ with a running byte counter (a `usage(did) += len` row updated + in the same transaction as the insert), not recomputed by scanning — e.g. + how object stores and Postgres-RLS SaaS backends bound tenants. xNet already + does this shape for backups/files; the `node_changes` path simply skips it. +- **Ephemeral demo environments.** The common pattern for public demo backends + (e.g. "playground" deployments) is a **scheduled full reset** (truncate all + tenant data on a cron / TTL) rather than per-user inactivity eviction, which + is exactly the "clear out all data every day" the operator expects. Railway + supports scheduled restarts/cron services; a persistent volume must be + actively truncated, not merely remounted. +- **Disk-full crash loops.** A full SQLite volume is a classic + crash-on-startup: `SQLITE_FULL` during migration/WAL checkpoint aborts boot, + the platform healthcheck fails, and you get a fallback 502 with no app-level + headers — see 0290. A disk-usage watchdog that sheds writes before the + volume fills prevents the hard-down. + +## Key Findings + +1. `--demo` is on, but demo mode enforces **nothing** on the data path. +2. The 10 MB cap is a **client-trusted hint**; the server caps only + backups/files, and at **1 GB**. +3. The `node_changes` append log is the **unbounded grower** and has no + per-DID accounting. +4. `EvictionService` is **dead code**: unwired, no storage impl, never + touched; and its inactivity semantics wouldn't clear an active user anyway. +5. The Railway volume is **persistent**, so nothing clears on restart. +6. This runaway is the **likely cause of the 502** in 0290. + +## Options And Tradeoffs + +### Bounding per-user size (the real 10 MB cap) + +| Option | What | Tradeoff | +| --- | --- | --- | +| **A. Enforce a per-DID byte budget on the write path** (recommended) | Maintain a `usage_by_did` counter updated in the same tx as `appendNodeChange`/`doc_state`; reject writes over `demoOverrides.quota`. | Real cap on the actual grower; needs a counter + reject signal back through node-relay (the client must handle rejection gracefully). | +| **B. Point Backup/File quota at the demo override** | In demo mode, pass `demoOverrides.quota`/`maxBlob` to Backup/File services. | Necessary but insufficient — doesn't touch `node_changes`, the main leak. Do it alongside A. | +| **C. Cap by row count / maxDocs** | Enforce `demoOverrides.maxDocs` (50) at the relay. | Coarse; a few huge docs still blow the byte budget. | + +### Clearing demo data ("daily") + +| Option | What | Tradeoff | +| --- | --- | --- | +| **A. Scheduled full reset every 24 h** (recommended for a demo) | A timer that truncates all demo tables (`node_changes`, `doc_state`, `doc_meta`, `backups`, `file_meta`, grants, share links) + `VACUUM`, gated on `config.demo`. Matches the operator's stated intent. | Wipes everyone (fine for a demo); must run in-process against the persistent volume, not rely on restart. | +| **B. Wire up the existing inactivity `EvictionService`** | Implement `EvictionStorage` in sqlite/memory, instantiate + `start()` in the lifecycle when demo, call `.touch(did)` on authenticated messages. | Reuses existing code, but **won't bound an active user** — only useful combined with the per-user quota (A above). | +| **C. Both** | Per-user quota bounds live size; daily reset caps long-term accumulation. | Most robust; a bit more code + tests. | + +### Safety net + +| Option | What | Tradeoff | +| --- | --- | --- | +| **Disk-usage watchdog** (recommended) | Before each write (or on a short interval), check volume usage; when >~85%, reject non-critical writes with a clear error instead of crashing. | Prevents the hard-down / 502 even if a future bug slips the quota. | + +## Recommendation + +1. **Immediate (unblock the disk + the 502):** stop the demo hub, **truncate + the demo data on the Railway volume** (delete `hub.db*` or `DELETE FROM + node_changes; VACUUM;` — demo data is disposable), redeploy, and confirm + `curl https://hub.xnet.fyi/health` → `200` and disk usage drops. This + should also clear 0290's outage if it's disk-full-induced. +2. **Enforce the real per-user cap (Option A + B):** add a per-DID byte counter + checked in `appendNodeChange`/`doc_state`, reject over `demoOverrides.quota`, + and route Backup/File quota to the demo override in demo mode. +3. **Implement a daily full reset (Clearing Option A):** a `config.demo`-gated + scheduled truncate-all + VACUUM, running in-process (persistent volume). + Prefer this over the inactivity `EvictionService` for the "clear daily" + requirement; wire eviction too only if inactivity cleanup is also wanted. +4. **Add a disk-usage watchdog** so a full volume sheds writes instead of + crashing. +5. **Close the test gap:** integration tests that (a) demo mode rejects writes + past 10 MB/DID on the `node_changes` path, (b) the daily reset empties all + demo tables, (c) a real storage backend satisfies whatever eviction/reset + interface ships. + +## Example Code + +Route quota to the demo override in demo mode (`packages/hub/src/server.ts:170`): + +```ts +const perUserQuota = config.demo && config.demoOverrides + ? config.demoOverrides.quota // 10 MB + : config.defaultQuota // 1 GB +const maxBlob = config.demo && config.demoOverrides + ? config.demoOverrides.maxBlob // 2 MB + : config.maxBlobSize +const backup = new BackupService(storage, { maxQuotaBytes: perUserQuota, maxBlobSize: maxBlob }) +const files = new FileService(storage, { maxStoragePerUser: perUserQuota }) +``` + +Per-DID byte budget on the change-log write path +(`packages/hub/src/services/node-relay.ts:140`): + +```ts +if (this.perUserQuota) { + const used = await this.storage.getUsageByDid(authorDid) // new + const incoming = byteLengthOf(change) + if (used + incoming > this.perUserQuota) { + return this.reject(peerId, 'quota-exceeded') // client shows a cap notice + } +} +await this.storage.appendNodeChange(room, change) +await this.storage.addUsageByDid(authorDid, byteLengthOf(change)) // same-tx counter +``` + +Daily reset (demo-gated), started in the lifecycle: + +```ts +if (config.demo) { + const resetMs = 24 * 60 * 60 * 1000 + setInterval(() => { + storage.truncateAllDemoData() // node_changes, doc_state, doc_meta, backups, file_meta, grants, share_links + VACUUM + .catch((e) => console.error('[demo-reset] failed', e)) + }, resetMs) +} +``` + +## Risks And Open Questions + +- **Rejecting a sync write mid-session:** the client must handle a + `quota-exceeded` relay rejection without data loss or a crash loop — it + should stop pushing and surface "demo storage full," keeping local data + intact. Needs a client-side path (ties into the handshake `demoLimits`). +- **Byte accounting accuracy:** counting `payload_json + signature` bytes vs. + on-disk page size will differ; the cap should target logical bytes with + headroom below the 500 MB physical volume. +- **VACUUM cost / Litestream:** on a demo hub, `VACUUM` after a daily + truncate is fine, but note the 0258 finding that Litestream VACUUM + invalidates lineage — the Railway path bypasses Litestream, so it's moot + here, but don't enable both without revisiting. +- **What counts as "a user" on a shared demo?** Quotas are per-DID; a single + human with multiple identities could still accumulate. Acceptable for a + demo, worth noting. +- **`private: true`** — `packages/hub` is private, so no changeset is required + for these fixes (per CLAUDE.md); still needs a Changelog fragment if the + repo's changelog check applies. + +## Implementation Checklist + +- [ ] **Immediate:** truncate demo data on the Railway volume; redeploy; confirm `/health` 200 and disk drops (also clears the 0290 502 if disk-full-induced). +- [x] Route Backup/File quota to `demoOverrides.quota`/`maxBlob` when `config.demo` (`server.ts:170-178`). +- [x] Add per-DID usage accounting to storage (`getUsageByDid`/`addUsageByDid` + a `usage_by_did` row/table) in `storage/sqlite.ts` and `storage/memory.ts`. +- [x] Enforce `demoOverrides.quota` on the `appendNodeChange` / `doc_state` write paths (`node-relay.ts`, `pool/node-pool.ts`); reject over budget. +- [x] Implement a `config.demo`-gated **daily truncate-all + VACUUM** started in the lifecycle (`server.ts`), operating on the persistent volume. +- [ ] (Optional) Implement `EvictionStorage` in sqlite/memory, instantiate + `start()` `EvictionService`, and call `.touch(did)` on authenticated messages — only if inactivity cleanup is also desired. +- [x] Add a disk-usage watchdog that sheds writes near capacity instead of crashing. +- [x] Handle a `quota-exceeded` relay rejection gracefully on the client (surface "demo storage full", keep local data). +- [x] Reconcile `litestream-entrypoint.sh:13` (missing `--demo`) with the Railway `startCommand` so all launch paths agree. +- [x] Tests: demo write over 10 MB/DID is rejected; daily reset empties all demo tables; real storage satisfies the reset/eviction interface. + +## Validation Checklist + +- [x] With `--demo`, a single DID syncing >10 MB is **rejected** at the relay; `node_changes` for that DID stays under budget. +- [ ] `hub.xnet.fyi` disk usage stays well under 500 MB across a day of use. +- [x] The daily reset empties `node_changes`/`doc_state`/backups/files and the volume shrinks (VACUUM), verified by `/health` `usedBytes`. +- [x] Backup/File uploads over 2 MB (demo `maxBlob`) return `413`/`BLOB_TOO_LARGE` in demo mode. +- [x] Killing available disk (simulated full volume) makes the hub **shed writes with a clear error**, not crash into a 502. +- [ ] `curl https://hub.xnet.fyi/health` returns 200 continuously (no crash loop) under sustained single-user load. + +## References + +- `railway.toml` — demo start command (`--demo --data /data`), healthcheck. +- `packages/hub/Dockerfile`, `packages/hub/litestream-entrypoint.sh` — image + entrypoint (Railway overrides `CMD`). +- `packages/hub/src/config.ts:82-140` — demo override resolution + `RAILWAY_VOLUME_MOUNT_PATH` data dir. +- `packages/hub/src/types.ts:97-138` — `DEFAULT_CONFIG` (1 GB) + `DEMO_DEFAULTS` (10 MB / 24 h). +- `packages/hub/src/server.ts:170-178,671-689,797-803` — quota wiring, lifecycle start (no eviction), handshake `demoLimits`. +- `packages/hub/src/services/node-relay.ts:129-160` — change ingestion (no quota) + `node-clear`. +- `packages/hub/src/storage/sqlite.ts:311-336,1141-1165,2089` — `node_changes` schema + `appendNodeChange`/`clearNodeChanges`. +- `packages/hub/src/services/eviction.ts`, `packages/hub/test/eviction.test.ts` — the dead service + its mock-only test. +- `packages/hub/src/services/backup.ts`, `services/files.ts` — the only real quota enforcers (set to 1 GB). +- Related explorations: 0290 (share-link failure / the 502 outage), 0258 (Cloud HA — Litestream/VACUUM lineage), and the cold-open-stall note (318k-row `changes` log). diff --git a/packages/hub/litestream-entrypoint.sh b/packages/hub/litestream-entrypoint.sh index 45d8590e2..b3fd871f4 100644 --- a/packages/hub/litestream-entrypoint.sh +++ b/packages/hub/litestream-entrypoint.sh @@ -12,6 +12,14 @@ PORT="${PORT:-4444}" CONFIG="${LITESTREAM_CONFIG:-/etc/litestream.yml}" HUB="node packages/hub/dist/cli.js --port ${PORT} --data ${DATA_DIR}" +# Demo hubs enforce per-user quotas + a daily data reset (exploration 0291). +# The config layer also treats HUB_MODE=demo as demo, but pass the flag through +# the Litestream path too so every launch path agrees (Railway overrides this +# start command with its own; this keeps the entrypoint consistent elsewhere). +if [ "$HUB_MODE" = "demo" ] || [ "$HUB_DEMO" = "1" ]; then + HUB="${HUB} --demo" +fi + # Managed hubs (Cloud Run) can't have a config file written into them, so generate # one from env when none is mounted: LITESTREAM=1, a per-tenant LITESTREAM_PATH, and # S3 creds. Credentials stay as ${...} refs so the rendered file never embeds diff --git a/packages/hub/src/config.ts b/packages/hub/src/config.ts index 03c3ef080..c2c696342 100644 --- a/packages/hub/src/config.ts +++ b/packages/hub/src/config.ts @@ -90,7 +90,10 @@ export const getDemoOverrides = (isDemo: boolean): DemoOverrides | null => { maxDocs: toNumber(process.env.DEMO_MAX_DOCS) ?? DEMO_DEFAULTS.maxDocs, maxBlob: toNumber(process.env.DEMO_MAX_BLOB) ?? DEMO_DEFAULTS.maxBlob, evictionTtl: toNumber(process.env.DEMO_EVICTION_TTL) ?? DEMO_DEFAULTS.evictionTtl, - evictionInterval: toNumber(process.env.DEMO_EVICTION_INTERVAL) ?? DEMO_DEFAULTS.evictionInterval + evictionInterval: + toNumber(process.env.DEMO_EVICTION_INTERVAL) ?? DEMO_DEFAULTS.evictionInterval, + resetInterval: toNumber(process.env.DEMO_RESET_INTERVAL) ?? DEMO_DEFAULTS.resetInterval, + diskLimitBytes: toNumber(process.env.DEMO_DISK_LIMIT) ?? DEMO_DEFAULTS.diskLimitBytes } } diff --git a/packages/hub/src/index.ts b/packages/hub/src/index.ts index 91ff9b74b..35639425c 100644 --- a/packages/hub/src/index.ts +++ b/packages/hub/src/index.ts @@ -4,6 +4,7 @@ import type { HubConfig, HubInstance } from './types' import { mkdirSync } from 'fs' +import { getDemoOverrides } from './config' import { createServer } from './server' import { DEFAULT_CONFIG } from './types' export { resolveConfig } from './config' @@ -37,6 +38,14 @@ export { createHubAuthError, type HubAuthError, type HubAuthErrorCode } from './ export const createHub = async (config: Partial = {}): Promise => { const resolved: HubConfig = { ...DEFAULT_CONFIG, ...config } + // `demo: true` must always carry enforceable limits: the CLI path resolves + // demoOverrides from env, but a programmatic `createHub({ demo: true })` + // used to leave them undefined — and every demo guardrail silently no-op'd + // (exploration 0291). Env vars still override the defaults here. + if (resolved.demo && !resolved.demoOverrides) { + resolved.demoOverrides = getDemoOverrides(true) ?? undefined + } + mkdirSync(resolved.dataDir, { recursive: true }) return createServer(resolved) diff --git a/packages/hub/src/pool/node-pool.ts b/packages/hub/src/pool/node-pool.ts index 57bf5aec6..2c1756a8d 100644 --- a/packages/hub/src/pool/node-pool.ts +++ b/packages/hub/src/pool/node-pool.ts @@ -16,6 +16,14 @@ type PoolEntry = { type NodePoolOptions = { maxWarmDocs?: number persistDelay?: number + /** + * When true, defer persisting doc state (the entry stays dirty and retries + * on the next markDirty/persistAll). Lets a demo hub ride out a full disk + * instead of crashing on SQLITE_FULL (exploration 0291). Yjs state has no + * per-DID attribution, so unlike node_changes it can't be quota'd per user — + * the watchdog is its only guard. + */ + isStorageFull?: () => boolean } const createEntry = (doc: Y.Doc): PoolEntry => ({ @@ -34,7 +42,7 @@ export class NodePool { constructor( private storage: HubStorage, - options?: NodePoolOptions + private options?: NodePoolOptions ) { this.maxWarmDocs = options?.maxWarmDocs ?? 500 this.persistDelay = options?.persistDelay ?? 1000 @@ -103,6 +111,13 @@ export class NodePool { const entry = this.entries.get(docId) if (!entry || !entry.dirty) return + // Full disk: keep the doc dirty in memory and try again on the next + // markDirty/persistAll rather than dying on SQLITE_FULL. + if (this.options?.isStorageFull?.()) { + entry.persistTimer = null + return + } + const state = Y.encodeStateAsUpdate(entry.doc) await this.storage.setDocState(docId, state) entry.dirty = false diff --git a/packages/hub/src/routes/share-links.ts b/packages/hub/src/routes/share-links.ts index f71a7fc03..e4e407785 100644 --- a/packages/hub/src/routes/share-links.ts +++ b/packages/hub/src/routes/share-links.ts @@ -32,7 +32,17 @@ export type ShareLinkRouteDeps = { // 'space' invites bootstrap Space membership: the grant a claim writes is keyed // on the Space id, so it acts as a container (subtree) grant that the hub // resolves for every node beneath the Space (exploration 0179). -const SHARE_DOC_TYPES = ['page', 'database', 'canvas', 'dashboard', 'view', 'space'] as const +// 'workspace' shares a saved shell layout — a bench travels like a node +// (exploration 0280; the client sent it long before the hub accepted it, 0290). +const SHARE_DOC_TYPES = [ + 'page', + 'database', + 'canvas', + 'dashboard', + 'view', + 'space', + 'workspace' +] as const type ShareDocType = (typeof SHARE_DOC_TYPES)[number] const isShareDocType = (value: unknown): value is ShareDocType => diff --git a/packages/hub/src/server.ts b/packages/hub/src/server.ts index 3d051aefc..c38a23dbd 100644 --- a/packages/hub/src/server.ts +++ b/packages/hub/src/server.ts @@ -53,6 +53,7 @@ import { FederationHealthChecker } from './services/federation-health' import { FileService } from './services/files' import { ShardRegistry } from './services/index-shards' import { KeyRegistryService } from './services/key-registry' +import { DiskWatchdog } from './services/disk-watchdog' import { NodeRelayService } from './services/node-relay' import { QueryService } from './services/query' import { RelayService } from './services/relay' @@ -156,7 +157,19 @@ export const createServer = async (config: HubConfig): Promise => { const storage = await createStorage(config.storage, config.dataDir, { resetOnCorruption: !!config.demo }) - const pool = new NodePool(storage) + // In demo mode, every per-user cap comes from the demo overrides (10 MB / + // 2 MB by default), not the 1 GB plan quota — otherwise a single visitor can + // fill the small demo volume (exploration 0291). + const demo = config.demo ? config.demoOverrides : undefined + const perUserQuota = demo ? demo.quota : config.defaultQuota + const maxBlobBytes = demo ? demo.maxBlob : config.maxBlobSize + // Demo-only: watch the data dir and shed relay writes before the volume fills + // (a full SQLite volume crashes the hub — exploration 0291 / the 0290 502). + const diskWatchdog = demo + ? new DiskWatchdog({ dataDir: config.dataDir, maxBytes: demo.diskLimitBytes }) + : null + const isStorageFull = diskWatchdog ? () => diskWatchdog.isFull() : undefined + const pool = new NodePool(storage, { isStorageFull }) const relayIdentity = generateIdentity() const relay = new RelayService(pool, { replication: config.sync, @@ -169,14 +182,14 @@ export const createServer = async (config: HubConfig): Promise => { } }) const backup = new BackupService(storage, { - maxQuotaBytes: config.defaultQuota, - maxBlobSize: config.maxBlobSize + maxQuotaBytes: perUserQuota, + maxBlobSize: maxBlobBytes }) // Files count against the same plan quota as backups (the hub's `defaultQuota`, // resolved from the signed HUB_PLAN entitlement). Without this, uploads fall back // to FileService's hardcoded 5 GiB default and silently diverge from the plan // quota the dashboard meter shows (exploration 0216). - const files = new FileService(storage, { maxStoragePerUser: config.defaultQuota }) + const files = new FileService(storage, { maxStoragePerUser: perUserQuota }) const keyRegistry = new KeyRegistryService() const taskIdentifiers = new TaskIdentifierService() const query = new QueryService(storage) @@ -259,7 +272,10 @@ export const createServer = async (config: HubConfig): Promise => { telemetry: config.telemetry, telemetryPeerHashSalt: config.telemetryPeerHashSalt } - const nodeRelay = new NodeRelayService(storage, remoteMutationTelemetry) + const nodeRelay = new NodeRelayService(storage, remoteMutationTelemetry, { + quotaBytes: demo ? demo.quota : undefined, + isStorageFull + }) const shareAccess = new ShareAccessService(storage) const awareness = new AwarenessService(storage, { ttlMs: config.awarenessTtlMs ?? 24 * 60 * 60 * 1000, @@ -692,6 +708,7 @@ export const createServer = async (config: HubConfig): Promise => { let httpServer: ReturnType | null = null let wss: WebSocketServer | null = null let sessionAuthInterval: ReturnType | null = null + let demoResetInterval: ReturnType | null = null const start = async (): Promise => { if (httpServer) return @@ -718,6 +735,20 @@ export const createServer = async (config: HubConfig): Promise => { await crawlCoordinator.seedUrls(crawlConfig.seedUrls) } } + // Demo hub: guard the small disposable volume — watch disk usage and wipe + // all user data on a fixed cadence so it can't grow unbounded (0291). + if (demo && diskWatchdog) { + diskWatchdog.start() + demoResetInterval = setInterval(() => { + storage + .resetAllUserData() + .then(({ nodeChanges, docStates }) => + console.log(`[demo-reset] wiped ${nodeChanges} node changes, ${docStates} doc states`) + ) + .catch((err) => console.error('[demo-reset] failed:', err)) + }, demo.resetInterval) + demoResetInterval.unref?.() + } await schemas.seedBuiltInSchemas([ { definition: PageSchema.schema, @@ -892,6 +923,11 @@ export const createServer = async (config: HubConfig): Promise => { clearInterval(sessionAuthInterval) sessionAuthInterval = null } + if (demoResetInterval) { + clearInterval(demoResetInterval) + demoResetInterval = null + } + diskWatchdog?.stop() if (wss) { for (const client of wss.clients) { diff --git a/packages/hub/src/services/disk-watchdog.ts b/packages/hub/src/services/disk-watchdog.ts new file mode 100644 index 000000000..8d05ee767 --- /dev/null +++ b/packages/hub/src/services/disk-watchdog.ts @@ -0,0 +1,82 @@ +/** + * @xnetjs/hub - Disk-usage watchdog (exploration 0291). + * + * Periodically measures the hub's on-disk footprint and flips a boolean when it + * crosses a fraction of the volume limit. The node relay consults `isFull()` and + * sheds writes (`STORAGE_FULL`) before the volume actually fills — a full SQLite + * volume otherwise crashes the process on the next write/checkpoint, which is how + * the demo hub went hard-down (see exploration 0290's 502). + * + * Sampling is cheap and coarse (a bounded `stat` walk), so it runs on an + * interval rather than per-write. + */ + +import { measureDataUsage, type DataUsageFs } from '../data-usage' + +export type DiskWatchdogOptions = { + /** Directory to measure (the hub data dir). */ + dataDir: string + /** Volume capacity in bytes to measure usage against. */ + maxBytes: number + /** Fraction of `maxBytes` at which writes start being shed (default 0.9). */ + threshold?: number + /** How often to re-measure, ms (default 30s). */ + checkIntervalMs?: number + /** Injectable fs for tests. */ + fs?: DataUsageFs +} + +export class DiskWatchdog { + private timer: ReturnType | null = null + private full = false + private readonly limitBytes: number + private readonly checkIntervalMs: number + + constructor(private options: DiskWatchdogOptions) { + const threshold = options.threshold ?? 0.9 + this.limitBytes = Math.max(0, options.maxBytes * threshold) + this.checkIntervalMs = options.checkIntervalMs ?? 30_000 + } + + /** Measure once and update the flag. Exposed for tests. */ + sample(): boolean { + const { usedBytes } = measureDataUsage(this.options.dataDir, this.options.fs) + const wasFull = this.full + this.full = usedBytes >= this.limitBytes + if (this.full && !wasFull) { + console.warn(`[disk-watchdog] usage ${usedBytes}B ≥ ${this.limitBytes}B — shedding writes`) + } else if (!this.full && wasFull) { + console.log(`[disk-watchdog] usage ${usedBytes}B back under limit — accepting writes`) + } + return this.full + } + + /** Whether the hub should currently shed writes. */ + isFull(): boolean { + return this.full + } + + start(): void { + this.stop() + this.sample() + this.timer = setInterval(() => { + try { + this.sample() + } catch (err) { + console.error('[disk-watchdog] sample failed:', err) + } + }, this.checkIntervalMs) + // Don't keep the process alive just for the watchdog. + this.timer.unref?.() + console.log( + `[disk-watchdog] started (limit=${this.limitBytes}B of ${this.options.maxBytes}B, every ${this.checkIntervalMs}ms)` + ) + } + + stop(): void { + if (this.timer) { + clearInterval(this.timer) + this.timer = null + } + } +} diff --git a/packages/hub/src/services/node-relay.ts b/packages/hub/src/services/node-relay.ts index 7102912d6..cc99e996b 100644 --- a/packages/hub/src/services/node-relay.ts +++ b/packages/hub/src/services/node-relay.ts @@ -57,7 +57,11 @@ export class NodeRelayError extends Error { | 'INVALID_CHANGE' | 'INVALID_SIGNATURE' | 'INVALID_HASH' - | 'REPLAY_REJECTED', + | 'REPLAY_REJECTED' + // The author's stored data would exceed the per-user cap (demo mode). + | 'QUOTA_EXCEEDED' + // The hub's disk is (near) full; writes are shed to avoid a crash. + | 'STORAGE_FULL', message: string, public action?: string, public resource?: string @@ -67,10 +71,30 @@ export class NodeRelayError extends Error { } } +export type NodeRelayOptions = { + /** + * Per-user storage cap in bytes (demo mode, exploration 0291). When set, a + * change is rejected if the author's existing `node_changes` bytes plus the + * incoming change would exceed it. Unset ⇒ unbounded (self-host default). + */ + quotaBytes?: number + /** + * Returns true when the hub's disk is at/near capacity. When it does, new + * changes are shed with `STORAGE_FULL` so a full volume degrades gracefully + * instead of crashing the process. + */ + isStorageFull?: () => boolean +} + +/** Serialized byte size a change contributes to a user's quota. */ +const changeUsageBytes = (change: SerializedNodeChange): number => + JSON.stringify(change.payload).length + change.signatureB64.length + export class NodeRelayService { constructor( private storage: HubStorage, - private telemetryOptions: RemoteMutationTelemetryOptions = {} + private telemetryOptions: RemoteMutationTelemetryOptions = {}, + private options: NodeRelayOptions = {} ) {} async handleNodeChange(msg: NodeChangeMessage, auth: AuthContext): Promise { @@ -137,6 +161,29 @@ export class NodeRelayService { } if (exists) return false + // Shed writes before the volume fills so a full disk degrades gracefully + // instead of crashing the hub (exploration 0291). + if (this.options.isStorageFull?.()) { + throw new NodeRelayError( + 'STORAGE_FULL', + 'Hub storage is full; new changes are temporarily rejected' + ) + } + + // Per-user storage cap (demo mode). The append-only change log is the + // primary grower and, unlike backups/files, had no quota gate — one active + // user could fill the disk (exploration 0291). + if (this.options.quotaBytes !== undefined) { + const used = await this.storage.getUsageBytesByDid(change.authorDID) + if (used + changeUsageBytes(msg.change) > this.options.quotaBytes) { + throw new NodeRelayError( + 'QUOTA_EXCEEDED', + `Storage limit reached (${this.options.quotaBytes} bytes per user). ` + + `Delete some data or use your own hub for more space.` + ) + } + } + await this.storage.appendNodeChange(msg.room, { ...msg.change, room: msg.room diff --git a/packages/hub/src/storage/interface.ts b/packages/hub/src/storage/interface.ts index d36c4e3aa..f5d5738cd 100644 --- a/packages/hub/src/storage/interface.ts +++ b/packages/hub/src/storage/interface.ts @@ -466,12 +466,27 @@ export type HubStorage = { getNodeChangesSince: (room: string, sinceLamport: number) => Promise getNodeChangesForNode: (room: string, nodeId: string) => Promise getHighWaterMark: (room: string) => Promise + /** + * Bytes of node-change data attributed to a DID (payload + signature), + * summed on demand. Backs the demo-mode per-user storage cap + * (exploration 0291) — the append-only `node_changes` log is the primary + * grower and, unlike backups/files, had no quota gate. + */ + getUsageBytesByDid: (did: string) => Promise /** * Delete every stored node-change for a room and return how many were * removed. Used by the "reset my data" dev tool — clearing a room is gated * on `hub/relay` for that room (you can only wipe rooms you can write to). */ clearNodeChanges: (room: string) => Promise + /** + * Wipe all user-content data (node changes, doc state, doc meta, database + * rows, blobs, files, grants, share links, containment/visibility, awareness) + * and return per-table counts. Backs the demo hub's scheduled daily reset + * (exploration 0291); leaves infrastructure (schemas, keys, peers, + * federation, shards) intact. + */ + resetAllUserData: () => Promise<{ nodeChanges: number; docStates: number }> // Database row operations insertDatabaseRow: (row: DatabaseRowRecord) => Promise diff --git a/packages/hub/src/storage/memory.ts b/packages/hub/src/storage/memory.ts index 084b20b4e..349c7bac6 100644 --- a/packages/hub/src/storage/memory.ts +++ b/packages/hub/src/storage/memory.ts @@ -690,6 +690,40 @@ export const createMemoryStorage = (): HubStorage => { return changes.length } + const getUsageBytesByDid = async (did: string): Promise => { + // Mirror the SQLite sum: LENGTH(payload_json) + LENGTH(signature_b64), + // where LENGTH on TEXT is a character count (exploration 0291). + let bytes = 0 + for (const change of nodeChangesByHash.values()) { + if (change.authorDid === did) { + bytes += JSON.stringify(change.payload).length + change.signatureB64.length + } + } + return bytes + } + + const resetAllUserData = async (): Promise<{ nodeChanges: number; docStates: number }> => { + const nodeChanges = nodeChangesByHash.size + const docStateCount = docStates.size + // User-content stores only — leave schemas/peers/federation/shards/crawlers + // intact so the hub keeps working after a reset (exploration 0291). + docStates.clear() + docMetas.clear() + searchBodies.clear() + docRecipients.clear() + blobs.clear() + files.clear() + grantsById.clear() + shareLinksById.clear() + nodeContainers.clear() + nodeVisibility.clear() + nodeChangesByHash.clear() + nodeChangesByRoom.clear() + databaseRows.clear() + awarenessByRoom.clear() + return { nodeChanges, docStates: docStateCount } + } + // ─── Database Row Operations ───────────────────────────────────────────────── const insertDatabaseRow = async (row: DatabaseRowRecord): Promise => { @@ -983,6 +1017,8 @@ export const createMemoryStorage = (): HubStorage => { listPopularSchemas, hasNodeChange, appendNodeChange, + getUsageBytesByDid, + resetAllUserData, getNodeChangesSince, getNodeChangesForNode, getHighWaterMark, diff --git a/packages/hub/src/storage/sqlite.ts b/packages/hub/src/storage/sqlite.ts index fea2a9c3e..21d55451f 100644 --- a/packages/hub/src/storage/sqlite.ts +++ b/packages/hub/src/storage/sqlite.ts @@ -334,6 +334,10 @@ const SCHEMA_SQL = ` ON node_changes(node_id, lamport_time); CREATE INDEX IF NOT EXISTS idx_node_changes_batch ON node_changes(batch_id) WHERE batch_id IS NOT NULL; + -- Backs getUsageBytesByDid: the demo per-user storage cap sums a DID's + -- change bytes on demand (exploration 0291). + CREATE INDEX IF NOT EXISTS idx_node_changes_author + ON node_changes(author_did); -- Database rows table for large database queries CREATE TABLE IF NOT EXISTS database_rows ( @@ -1160,6 +1164,10 @@ export const createSQLiteStorage = ( getHighWaterMark: db.prepare(` SELECT MAX(lamport_time) as hwm FROM node_changes WHERE room = ? `), + getUsageBytesByDid: db.prepare(` + SELECT COALESCE(SUM(LENGTH(payload_json) + LENGTH(signature_b64)), 0) AS bytes + FROM node_changes WHERE author_did = ? + `), clearNodeChanges: db.prepare(` DELETE FROM node_changes WHERE room = ? `), @@ -2135,6 +2143,47 @@ export const createSQLiteStorage = ( return info.changes } + const getUsageBytesByDid = async (did: string): Promise => { + const row = stmts.getUsageBytesByDid.get(did) as { bytes: number } | undefined + return row?.bytes ?? 0 + } + + // User-content tables wiped by the demo daily reset (exploration 0291). + // Infrastructure (schemas, keys, peers, federation, shards, crawlers) is + // intentionally excluded so the hub keeps working after a reset. FTS mirrors + // (search_index←doc_meta, database_rows_fts←database_rows) are external-content + // tables kept in sync by AFTER DELETE triggers, so deleting the base rows + // clears them too. + const DEMO_RESET_TABLES = [ + 'node_changes', + 'doc_state', + 'doc_meta', + 'doc_recipients', + 'database_rows', + 'grant_index', + 'share_links', + 'node_container', + 'node_visibility', + 'awareness_state', + 'backups', + 'file_meta' + ] as const + + const resetAllUserData = async (): Promise<{ nodeChanges: number; docStates: number }> => { + const countOf = (table: string): number => + (db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n + const nodeChanges = countOf('node_changes') + const docStates = countOf('doc_state') + const wipe = db.transaction(() => { + for (const table of DEMO_RESET_TABLES) db.prepare(`DELETE FROM ${table}`).run() + }) + wipe() + // VACUUM cannot run inside a transaction — reclaim the freed pages so the + // on-disk file actually shrinks (the whole point of the daily reset). + db.exec('VACUUM') + return { nodeChanges, docStates } + } + const close = async (): Promise => { db.close() } @@ -2515,6 +2564,8 @@ export const createSQLiteStorage = ( listPopularSchemas, hasNodeChange, appendNodeChange, + getUsageBytesByDid, + resetAllUserData, getNodeChangesSince, getNodeChangesForNode, getHighWaterMark, diff --git a/packages/hub/src/types.ts b/packages/hub/src/types.ts index 910a79c7a..539408c9a 100644 --- a/packages/hub/src/types.ts +++ b/packages/hub/src/types.ts @@ -127,6 +127,10 @@ export type DemoOverrides = { evictionTtl: number /** How often to run eviction check (ms). Default: 1 hour. */ evictionInterval: number + /** Wipe all user data on this cadence (ms). Default: 24 hours. */ + resetInterval: number + /** Volume capacity the disk watchdog guards (bytes). Default: 500 MB. */ + diskLimitBytes: number } export const DEMO_DEFAULTS: DemoOverrides = { @@ -134,7 +138,9 @@ export const DEMO_DEFAULTS: DemoOverrides = { maxDocs: 50, maxBlob: 2 * 1024 * 1024, // 2 MB evictionTtl: 24 * 60 * 60 * 1000, // 24 hours - evictionInterval: 60 * 60 * 1000 // 1 hour + evictionInterval: 60 * 60 * 1000, // 1 hour + resetInterval: 24 * 60 * 60 * 1000, // 24 hours + diskLimitBytes: 500 * 1024 * 1024 // 500 MB (Railway demo volume) } export type HubInstance = { diff --git a/packages/hub/test/demo-enforcement.test.ts b/packages/hub/test/demo-enforcement.test.ts new file mode 100644 index 000000000..30d744cd5 --- /dev/null +++ b/packages/hub/test/demo-enforcement.test.ts @@ -0,0 +1,231 @@ +/** + * Demo-mode storage guardrails (exploration 0291). + * + * These are the integration tests that were missing: the per-user quota and + * the daily reset are exercised against REAL storage + the real relay, not a + * mock. That gap is why the demo hub silently grew past its 500 MB volume. + */ +import type { AuthContext } from '../src/auth/ucan' +import type { HubStorage, SerializedNodeChange } from '../src/storage/interface' +import type { DID } from '@xnetjs/core' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { bytesToBase64, generateSigningKeyPair } from '@xnetjs/crypto' +import { identityFromPrivateKey } from '@xnetjs/identity' +import { createChangeId, createUnsignedChange, signChange } from '@xnetjs/sync' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { DiskWatchdog } from '../src/services/disk-watchdog' +import { NodeRelayError, NodeRelayService } from '../src/services/node-relay' +import { createMemoryStorage } from '../src/storage/memory' +import { createSQLiteStorage } from '../src/storage/sqlite' + +const ROOM = 'demo-room' + +// A stable software identity so every change is attributed to one DID and the +// per-user cap actually accumulates. +const { privateKey } = generateSigningKeyPair() +const identity = identityFromPrivateKey(privateKey) + +const makeSignedChange = (nodeId: string, lamport: number): SerializedNodeChange => { + const payload = { + nodeId, + schemaId: 'xnet://xnet.dev/Task', + properties: { title: `Task ${nodeId}`, status: 'todo' } + } + const unsigned = createUnsignedChange({ + id: createChangeId(), + type: 'node-change', + payload, + parentHash: null, + authorDID: identity.did as DID, + wallTime: 1_700_000_000_000 + lamport, + lamport + }) + const signed = signChange(unsigned, privateKey) + return { + id: signed.id, + type: signed.type, + hash: signed.hash, + room: ROOM, + nodeId, + schemaId: payload.schemaId, + lamportTime: signed.lamport, + lamportAuthor: signed.authorDID, + authorDid: signed.authorDID, + wallTime: signed.wallTime, + parentHash: signed.parentHash, + payload: signed.payload, + signatureB64: bytesToBase64(signed.signature), + protocolVersion: signed.protocolVersion, + batchId: signed.batchId, + batchIndex: signed.batchIndex, + batchSize: signed.batchSize + } +} + +const usageOf = (change: SerializedNodeChange): number => + JSON.stringify(change.payload).length + change.signatureB64.length + +// `handleNodeChange` only reads `.did` and `.can(...)`. +const allowAuth = { did: identity.did, can: () => true } as unknown as AuthContext + +const relayMsg = (change: SerializedNodeChange) => + ({ type: 'node-change', room: ROOM, change }) as const + +// ─── Storage: per-DID usage + reset (both backends) ───────────────────────── + +let sqliteAvailable = false +try { + const probe = mkdtempSync(join(tmpdir(), 'hub-probe-')) + createSQLiteStorage(probe).close() + rmSync(probe, { recursive: true, force: true }) + sqliteAvailable = true +} catch { + sqliteAvailable = false +} + +type Factory = { name: string; create: () => { storage: HubStorage; cleanup: () => void } } +const factories: Factory[] = [ + { name: 'Memory', create: () => ({ storage: createMemoryStorage(), cleanup: () => {} }) }, + ...(sqliteAvailable + ? [ + { + name: 'SQLite', + create: () => { + const dir = mkdtempSync(join(tmpdir(), 'hub-demo-')) + return { + storage: createSQLiteStorage(dir), + cleanup: () => rmSync(dir, { recursive: true, force: true }) + } + } + } + ] + : []) +] + +describe.each(factories)('demo storage accounting ($name)', ({ create }) => { + let storage: HubStorage + let cleanup: () => void + beforeEach(() => { + const created = create() + storage = created.storage + cleanup = created.cleanup + }) + afterEach(async () => { + await storage.close?.() + cleanup() + }) + + it('getUsageBytesByDid sums a DID change bytes and ignores others', async () => { + const c1 = makeSignedChange('node-1', 1) + const c2 = makeSignedChange('node-2', 2) + await storage.appendNodeChange(ROOM, c1) + await storage.appendNodeChange(ROOM, c2) + + const used = await storage.getUsageBytesByDid(identity.did) + expect(used).toBe(usageOf(c1) + usageOf(c2)) + expect(await storage.getUsageBytesByDid('did:key:zNobody')).toBe(0) + }) + + it('resetAllUserData wipes user content and returns counts', async () => { + await storage.appendNodeChange(ROOM, makeSignedChange('node-1', 1)) + await storage.appendNodeChange(ROOM, makeSignedChange('node-2', 2)) + await storage.setDocState('doc-1', new Uint8Array([1, 2, 3])) + + const result = await storage.resetAllUserData() + expect(result.nodeChanges).toBe(2) + expect(result.docStates).toBe(1) + + expect(await storage.getUsageBytesByDid(identity.did)).toBe(0) + expect(await storage.getNodeChangesSince(ROOM, 0)).toHaveLength(0) + expect(await storage.getDocState('doc-1')).toBeNull() + }) +}) + +// ─── Relay: quota + storage-full enforcement ──────────────────────────────── + +describe('node relay demo enforcement', () => { + it('rejects a change that would exceed the per-user quota', async () => { + const storage = createMemoryStorage() + const first = makeSignedChange('node-1', 1) + // Budget exactly one change: the second (same-size) change must exceed it. + const relay = new NodeRelayService(storage, {}, { quotaBytes: usageOf(first) }) + + await expect(relay.handleNodeChange(relayMsg(first), allowAuth)).resolves.toBe(true) + + const second = makeSignedChange('node-2', 2) + await expect(relay.handleNodeChange(relayMsg(second), allowAuth)).rejects.toMatchObject({ + code: 'QUOTA_EXCEEDED' + }) + // The rejected change was not stored. + expect(await storage.hasNodeChange(second.hash)).toBe(false) + }) + + it('allows unbounded writes when no quota is configured (self-host default)', async () => { + const storage = createMemoryStorage() + const relay = new NodeRelayService(storage, {}, {}) + for (let i = 0; i < 5; i++) { + await expect( + relay.handleNodeChange(relayMsg(makeSignedChange(`node-${i}`, i + 1)), allowAuth) + ).resolves.toBe(true) + } + }) + + it('sheds writes with STORAGE_FULL when the disk is full', async () => { + const storage = createMemoryStorage() + const relay = new NodeRelayService(storage, {}, { isStorageFull: () => true }) + await expect( + relay.handleNodeChange(relayMsg(makeSignedChange('node-1', 1)), allowAuth) + ).rejects.toBeInstanceOf(NodeRelayError) + await expect( + relay.handleNodeChange(relayMsg(makeSignedChange('node-1', 1)), allowAuth) + ).rejects.toMatchObject({ code: 'STORAGE_FULL' }) + }) +}) + +// ─── createHub demo resolution ────────────────────────────────────────────── + +describe('createHub demo overrides', () => { + it('resolves demoOverrides when demo: true is passed programmatically', async () => { + // The CLI resolves demoOverrides from env, but createHub({ demo: true }) + // used to leave them undefined — silently disabling every guardrail. + const { createHub } = await import('../src') + const hub = await createHub({ port: 14971, demo: true, storage: 'memory' }) + expect(hub.config.demoOverrides).toBeDefined() + expect(hub.config.demoOverrides?.quota).toBeGreaterThan(0) + await hub.stop() + }) +}) + +// ─── Disk watchdog ────────────────────────────────────────────────────────── + +describe('DiskWatchdog', () => { + const fsWith = (bytes: number) => ({ + readdir: () => ['hub.db'], + stat: () => ({ isDirectory: () => false, isFile: () => true, size: bytes, mtimeMs: 1 }) + }) + + it('flips isFull when usage crosses the threshold', () => { + const wd = new DiskWatchdog({ + dataDir: '/data', + maxBytes: 500 * 1024 * 1024, + threshold: 0.9, + fs: fsWith(600 * 1024 * 1024) + }) + expect(wd.isFull()).toBe(false) // not sampled yet + wd.sample() + expect(wd.isFull()).toBe(true) + }) + + it('stays clear while under the threshold', () => { + const wd = new DiskWatchdog({ + dataDir: '/data', + maxBytes: 500 * 1024 * 1024, + threshold: 0.9, + fs: fsWith(100 * 1024 * 1024) + }) + wd.sample() + expect(wd.isFull()).toBe(false) + }) +}) diff --git a/packages/hub/test/share-links.test.ts b/packages/hub/test/share-links.test.ts index 153b8a433..ac9c73b1f 100644 --- a/packages/hub/test/share-links.test.ts +++ b/packages/hub/test/share-links.test.ts @@ -218,6 +218,27 @@ describe('Share Links', () => { expect(JSON.stringify(links[0])).not.toContain(url.split('#s=')[1]) }) + it('accepts every client ShareDocType, including workspace (0290)', async () => { + // The client offered 'workspace' (saved bench, 0280) long before the hub + // accepted it — every union member must round-trip create → claim. + const docTypes = ['page', 'database', 'canvas', 'dashboard', 'view', 'space', 'workspace'] + for (const docType of docTypes) { + const { status, json } = await api('/shares/links', { + method: 'POST', + token: owner.token, + body: { docId: `doc-type-${docType}`, docType, role: 'read' } + }) + expect(status, `docType=${docType}`).toBe(200) + expect(json.docType).toBe(docType) + + const recipient = makeActor() + const secret = (json.url as string).split('#s=')[1] + const claimed = await claim(recipient, json.linkId as string, secret) + expect(claimed.status, `claim docType=${docType}`).toBe(200) + expect(claimed.json.docType).toBe(docType) + } + }) + it('claims a link, records a grant, and is idempotent on re-claim', async () => { const recipient = makeActor() const { linkId, secret } = await createLink(owner, 'doc-claim', 'read') diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index eed44a7aa..3cc9fbc56 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -100,7 +100,9 @@ export { export { NodeStoreSyncProvider, type SerializedNodeChange, - type NodeSyncResponse + type NodeSyncResponse, + type SyncBlockedListener, + type SyncBlockedReason } from './sync/node-store-sync-provider' export { diff --git a/packages/runtime/src/sync/node-store-sync-provider.test.ts b/packages/runtime/src/sync/node-store-sync-provider.test.ts index 4cd967c54..60ba50613 100644 --- a/packages/runtime/src/sync/node-store-sync-provider.test.ts +++ b/packages/runtime/src/sync/node-store-sync-provider.test.ts @@ -565,4 +565,66 @@ describe('NodeStoreSyncProvider', () => { warn.mockRestore() }) }) + + describe('capacity halt (0291 demo quota / disk full)', () => { + it('halts outbound on the FIRST QUOTA_EXCEEDED and notifies listeners', async () => { + const { store, emit } = makeStore() + const { conn, setStatus, injectMessage } = makeConnection('connected') + const provider = new NodeStoreSyncProvider(store, 'room-1') + provider.attach(conn) + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const blocked = vi.fn() + provider.onSyncBlocked(blocked) + + // One rejection is enough: the account stays over quota for every + // subsequent change, so resending only floods the hub. + injectMessage({ type: 'node-error', code: 'QUOTA_EXCEEDED', error: 'over 10MB' }) + expect(error).toHaveBeenCalledTimes(1) + expect(blocked).toHaveBeenCalledWith('QUOTA_EXCEEDED', 'over 10MB') + + // Local changes are still accepted locally but NOT published while halted. + emit({ change: makeChange(1), isRemote: false }) + await vi.advanceTimersByTimeAsync(0) + expect(conn.publish).not.toHaveBeenCalled() + + // Reconnect clears the halt (a demo reset / freed disk lifts the cap). + setStatus('disconnected') + setStatus('connected') + await vi.advanceTimersByTimeAsync(0) + injectMessage({ type: 'node-sync-response', room: 'room-1', changes: [], highWaterMark: 0 }) + await vi.advanceTimersByTimeAsync(0) + emit({ change: makeChange(2), isRemote: false }) + await vi.advanceTimersByTimeAsync(0) + expect(conn.publish).toHaveBeenCalledTimes(1) + + error.mockRestore() + warn.mockRestore() + }) + + it('halts outbound on STORAGE_FULL and unsubscribes listeners cleanly', async () => { + const { store, emit } = makeStore() + const { conn, injectMessage } = makeConnection('connected') + const provider = new NodeStoreSyncProvider(store, 'room-1') + provider.attach(conn) + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const blocked = vi.fn() + const unsubscribe = provider.onSyncBlocked(blocked) + + injectMessage({ type: 'node-error', code: 'STORAGE_FULL', error: 'disk full' }) + expect(blocked).toHaveBeenCalledWith('STORAGE_FULL', 'disk full') + + unsubscribe() + injectMessage({ type: 'node-error', code: 'STORAGE_FULL', error: 'disk full' }) + expect(blocked).toHaveBeenCalledTimes(1) + + emit({ change: makeChange(1), isRemote: false }) + await vi.advanceTimersByTimeAsync(0) + expect(conn.publish).not.toHaveBeenCalled() + + error.mockRestore() + warn.mockRestore() + }) + }) }) diff --git a/packages/runtime/src/sync/node-store-sync-provider.ts b/packages/runtime/src/sync/node-store-sync-provider.ts index 45f34415b..d4e9fe200 100644 --- a/packages/runtime/src/sync/node-store-sync-provider.ts +++ b/packages/runtime/src/sync/node-store-sync-provider.ts @@ -93,6 +93,17 @@ export type NodeSyncResponse = { */ export type UnknownChangeTypeListener = (change: NodeChange, peerId: string) => void +/** + * The hub is refusing further writes for a capacity reason (exploration 0291): + * `QUOTA_EXCEEDED` — this identity is over the hub's per-user cap (demo mode); + * `STORAGE_FULL` — the hub's volume is full and it is shedding writes. + * Local data is untouched; outbound sync pauses until the next reconnect. + */ +export type SyncBlockedReason = 'QUOTA_EXCEEDED' | 'STORAGE_FULL' +export type SyncBlockedListener = (reason: SyncBlockedReason, detail: string) => void + +const CAPACITY_REJECTION_CODES = new Set(['QUOTA_EXCEEDED', 'STORAGE_FULL']) + export class NodeStoreSyncProvider { /** Confirmed, persisted high-water mark (advanced from the hub's response). */ private lastSyncedLamport = 0 @@ -106,6 +117,7 @@ export class NodeStoreSyncProvider { private messageCleanup: (() => void) | null = null private storeCleanup: (() => void) | null = null private unknownChangeTypeListeners = new Set() + private syncBlockedListeners = new Set() // Throttled send queue. private sendQueue: NodeChange[] = [] @@ -338,6 +350,9 @@ export class NodeStoreSyncProvider { if (STRUCTURAL_REJECTION_CODES.has(code)) { this.recordStructuralRejection(code, message) } + if (CAPACITY_REJECTION_CODES.has(code as SyncBlockedReason)) { + this.recordCapacityRejection(code as SyncBlockedReason, message) + } return } if (message.type !== 'node-sync-response') return @@ -373,6 +388,50 @@ export class NodeStoreSyncProvider { ) } + /** + * Halt outbound sync on a capacity rejection (exploration 0291). Unlike the + * structural breaker, one rejection is enough: while the account is over + * quota (or the hub's disk is full) every further change is rejected too, so + * resending only floods the hub. Local data is untouched — the store keeps + * accepting writes and the un-pushed changes replay on the next reconnect + * (the demo hub's daily reset / freed disk clears the condition server-side). + */ + private recordCapacityRejection( + reason: SyncBlockedReason, + message: Record + ): void { + const detail = String(message.error ?? message.message ?? '') + if (!this.outboundHalted) { + this.outboundHalted = true + this.clearSendQueue() + console.error( + reason === 'QUOTA_EXCEEDED' + ? `[NodeStoreSync] Hub storage limit reached — pausing outbound sync. Your data is safe ` + + `locally; syncing resumes when space frees up (demo hubs reset daily). Hub said: ${detail}` + : `[NodeStoreSync] Hub disk is full — pausing outbound sync. Your data is safe locally; ` + + `syncing resumes automatically. Hub said: ${detail}` + ) + } + for (const listener of this.syncBlockedListeners) { + try { + listener(reason, detail) + } catch (err) { + console.error('Error in sync-blocked listener:', err) + } + } + } + + /** + * Subscribe to capacity-blocked events (hub over quota / disk full) so the + * app can surface a "storage full" notice. Returns an unsubscribe function. + */ + onSyncBlocked(listener: SyncBlockedListener): () => void { + this.syncBlockedListeners.add(listener) + return () => { + this.syncBlockedListeners.delete(listener) + } + } + private requestSync(): void { if (!this.connection) return this.connection.sendRaw({ diff --git a/site/src/data/changelog/2026-07-10-demo-hub-now-enforces-its-storage-limits.json b/site/src/data/changelog/2026-07-10-demo-hub-now-enforces-its-storage-limits.json new file mode 100644 index 000000000..42b207b7d --- /dev/null +++ b/site/src/data/changelog/2026-07-10-demo-hub-now-enforces-its-storage-limits.json @@ -0,0 +1,11 @@ +{ + "id": "2026-07-10-demo-hub-now-enforces-its-storage-limits", + "date": "July 10, 2026", + "title": "Demo hub now enforces its storage limits", + "summary": "The shared demo hub enforces its 10 MB per-user cap, wipes demo data daily, and sheds writes before its disk fills instead of crashing. Workspace (bench) share links now work end-to-end, and a hub that is down reports 'isn't responding' instead of a cryptic 'Failed to fetch'.", + "highlights": [], + "tags": [ + "sync", + "platform" + ] +}