diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 504de9bbdc..7bd1099f08 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -166,13 +166,15 @@ migration-safe chunked interface: with `section: "storage_runner"` and a `storage_id`, using the same StorageRunner `exportStorage({ pageSize, startAfter })` RPC as the dedicated storage export capability. User meter counters use `section: "user_meter"` and - the `UserMeter.exportCounters` RPC. R2 raw MIME, attachment, avatar, and icon - objects use `section: "r2_object"`; each response contains at most one 256 KiB - base64 chunk and an opaque cursor. Each request uses bounded `LIMIT 1` - ownership queries rather than reconstructing inventory. Continuation cursors - bind the source row, object key, size, and ETag; ownership/key mutations and - object overwrites are reported instead of mixing generations. Missing objects - are represented explicitly. + the `UserMeter.exportCounters` RPC (daily counters plus additive + `storageBytesShadow` on the first page only when present; explicitly + non-authoritative). R2 raw MIME, attachment, avatar, and icon objects use + `section: "r2_object"`; each response contains at most one 256 KiB base64 + chunk and an opaque cursor. Each request uses bounded `LIMIT 1` ownership + queries rather than reconstructing inventory. Continuation cursors bind the + source row, object key, size, and ETag; ownership/key mutations and object + overwrites are reported instead of mixing generations. Missing objects are + represented explicitly. D1 manifest counts use bounded SQL `COUNT(*)` queries. D1 section rows are read with SQL-level keyset pagination: every query orders by the table's `rowid`, @@ -196,8 +198,13 @@ Durable Object export behavior: never pruned by retention. See [Run records](./run-records.md). - `UserMeter` exports daily entitlement counter rows through the `user_meter` section (`exportCounters` RPC; keyset pagination by UTC `day` and `resource`). - Retention is self-enforced inside the DO (seven UTC days of counter and - inbound-delivery-claim rows); see + The same RPC may return additive `storageBytesShadow` on the first page only + (`startAfter` absent) when the schema-v4 shadow row exists (`null` on later + pages and when never shadowed); section totals count it as one row when + present, but it is explicitly **non-authoritative** — usage and enforcement + read D1 `users.d1_storage_bytes`. Retention is self-enforced inside the DO + (seven UTC days of counter and inbound-delivery-claim rows); shadow + storage-byte state is not time-pruned. See [Entitlements](./entitlements.md#usermeter-expand-phase). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. @@ -238,11 +245,15 @@ The schema is defined by migrations in `packages/worker/migrations/`: default `public`) come from migration 0068. `account_type` (`'person'` default or `'platform'`, migration 0072) distinguishes normal signups from operator-provisioned platform accounts that own official package scopes (see - [Platform accounts](./platform-accounts.md)). Inbound email routing does not - reverse-resolve stable ids at all — it uses the indexed username lookup - (`findPublicUserIdentityByUsername`). Contextless paths resolve stable ids - with one indexed point read on `users.stable_user_id` (for example - `findUserAccountByStableUserId`). + [Platform accounts](./platform-accounts.md)). `d1_storage_bytes` and + `d1_storage_bytes_updated_at` (migration 0122) are the **sole authority** for + D1 payload storage-byte read, enforcement, and reconciliation. UserMeter + `storage_bytes_state` (schema v4) is an optional expand-phase shadow only — + see [Entitlements](./entitlements.md#usermeter-expand-phase). Inbound email + routing does not reverse-resolve stable ids at all — it uses the indexed + username lookup (`findPublicUserIdentityByUsername`). Contextless paths + resolve stable ids with one indexed point read on `users.stable_user_id` (for + example `findUserAccountByStableUserId`). - `platform_feedback`: attributed, user-approved Kody feedback and admin triage state. Submitter identity remains on the row; optional reviewer attribution is cleared if that admin account is deleted. Open and triaged rows remain until @@ -489,16 +500,20 @@ Storage split: Daily rate-style entitlement counters and inbound email delivery-id idempotency live in a per-user `UserMeter` Durable Object with SQLite -(`packages/worker/src/entitlements/user-meter-do.ts`). The Worker binding is -`USER_METER` (class `UserMeter`; Wrangler SQLite migration tag `v21` via -`new_sqlite_classes` in `packages/worker/wrangler.jsonc`). +(`packages/worker/src/entitlements/user-meter-do.ts`). Schema v4 adds an +optional `storage_bytes_state` singleton as a **best-effort shadow** of D1 +`users.d1_storage_bytes` for future cutover — D1 remains sole authority for +reads, reserves, and reconciliation in the current additive slice. The Worker +binding is `USER_METER` (class `UserMeter`; Wrangler SQLite migration tag `v21` +via `new_sqlite_classes` in `packages/worker/wrangler.jsonc`). Naming matches `RunLog` and `JobManager`: one object per untrimmed stable MCP `userId` via `userMeterDurableObjectName(userId)` → `idFromName(userId)` in `packages/worker/src/user-scoped-durable-object-name.ts`. There is no `user_id` column inside the DO because the object identity is the user. -SQLite ownership (schema version tracked in `user_meter_meta`): +SQLite ownership (schema version tracked in `user_meter_meta`; current version +**4**): - `daily_counters` — authoritative UTC-day counters for `email_sends_per_day`, `email_receives_per_day`, `execute_calls_per_day`, and @@ -509,30 +524,41 @@ SQLite ownership (schema version tracked in `user_meter_meta`): claim's resource/day, post-charge counter, revision, and `claimed_at` so Cloudflare Email Routing retries inside the 48-hour inbound dedupe window cannot double-charge `email_receives_per_day`. +- `storage_bytes_state` — singleton **shadow** of D1 payload bytes (`id = 1` + CHECK constraint, `bytes`, monotonic `revision`, `updated_at`). Added in + schema v4. Populated by optional non-awaited shadow writes after D1 reserves + and optional reconcile shadows; never read for enforcement or usage display. + StorageRunner bucket estimates stay outside this row (see + [Entitlements](./entitlements.md#usermeter-expand-phase)). Retention is self-enforced inside the DO: every read/write path opportunistically deletes counter and claim rows older than seven UTC days (`userMeterDailyCounterRetentionDays`). Enforcement only needs the current day; the window covers timezone edge cases, recent account exports, and inbound -retries. - -**Expand-phase D1 mirror:** enforcement and point reads are authoritative in -UserMeter. D1 `entitlement_daily_counters` is **not** dropped — it remains a -best-effort mirror for existing readers and reporting. After each DO -consume/refund/inbound claim, the entitlements service schedules a non-awaited -absolute mirror write keyed by `(user_id, resource, day)` with a -revision-ordered `updated_at` token (`r/` + zero-padded revision from -`userMeterMirrorUpdatedAtToken`) so late writes cannot overwrite newer state. -See [Entitlements](./entitlements.md#usermeter-expand-phase). - -**Cold bootstrap:** a missing `(resource, day)` row returns `needs_bootstrap`. -The service performs one legacy D1 point read on `entitlement_daily_counters`, -then `initialize()` seeds the DO row with `INSERT OR IGNORE` (concurrent callers -cannot double-apply the baseline). Warm paths never read D1 for enforcement. - -Account deletion calls `UserMeter.purge()` (one RPC per user, no D1 id scan). -Account export pages `UserMeter.exportCounters` through the `user_meter` -manifest section / `account_export_section`. +retries. Shadow storage-byte state is not time-pruned. + +**Expand-phase D1 mirrors (daily counters only):** enforcement and point reads +are authoritative in UserMeter for daily counters. D1 +`entitlement_daily_counters` is **not** dropped — it remains a best-effort +mirror for existing readers and reporting. After each DO consume/refund/inbound +claim, the entitlements service schedules a non-awaited absolute mirror write +keyed by `(user_id, resource, day)` with a revision-ordered `updated_at` token +(`r/` + zero-padded revision from `userMeterMirrorUpdatedAtToken`) so late +writes cannot overwrite newer state. See +[Entitlements](./entitlements.md#usermeter-expand-phase). + +**Daily cold bootstrap:** a missing `(resource, day)` row returns +`needs_bootstrap`. The service performs one legacy D1 point read on +`entitlement_daily_counters`, then `initialize()` seeds the DO row with +`INSERT OR IGNORE` (concurrent callers cannot double-apply the baseline). Warm +daily paths never read D1 for enforcement. + +Account deletion calls `UserMeter.purge()` (one RPC per user, no D1 id scan; +`deleteAll` clears counters, claims, and any shadow storage state). Account +export pages `UserMeter.exportCounters` through the `user_meter` manifest +section / `account_export_section` (daily counters plus additive +`storageBytesShadow` on the first page only when present; shadow field is +non-authoritative). ### Package state model @@ -570,8 +596,9 @@ via `durableObjectNameFromParts`); domain helpers such as and dedicated unpruned state (workflow projections, job-run observability, package activation counters/milestones). See [Run records](./run-records.md). - `UserMeter` — `userMeterDurableObjectName(userId)` → `idFromName(userId)`. One - daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`). - See [Entitlements](./entitlements.md#usermeter-expand-phase). + daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`), + plus optional schema-v4 D1 storage-byte shadow. See + [Entitlements](./entitlements.md#usermeter-expand-phase). - `McpClientHub` — `mcpClientHubDurableObjectName(userId)` → `idFromName(userId.trim())`. - `StorageRunner` — `storageRunnerDurableObjectName(userId, storageId)` → @@ -886,7 +913,7 @@ to `durableObjectNameFromParts`). - `JobManager`: `idFromName(userId)` (no trim). - `RunLog`: `idFromName(userId)` (no trim); one execution-history DO per user. - `UserMeter`: `idFromName(userId)` (no trim); one daily-entitlement meter DO - per user. + per user, plus optional schema-v4 D1 storage-byte shadow. - `McpClientHub`: `idFromName(userId.trim())`. - `StorageRunner`: `idFromName(JSON.stringify([userId, storageId]))`. - `RepoSession`: `idFromName(repo_sessions.id)`; the key is not user-prefixed, diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index c4c983ba17..7171beca42 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -119,6 +119,17 @@ per-user `UserMeter` Durable Object** (`USER_METER` binding). Code lives in `packages/worker/src/entitlements/user-meter-do.ts` and `user-meter-client.ts`; storage layout and naming are documented in [Data storage](./data-storage.md). +**D1 payload storage bytes** (`storage_bytes`) stay authoritative on D1 +`users.d1_storage_bytes` (migration 0122). UserMeter schema v4 adds an optional +`storage_bytes_state` singleton as a **best-effort shadow** for future cutover — +it does not drive reads, reserves, or reconciliation in this additive slice. + +StorageRunner bucket `estimatedBytes` and the per-bucket inventory in +`user_storage_buckets` stay a **separate** quota component. StorageRunner write +chokepoints pass `getCurrent` as a check-only composed total (D1 payload bytes +from `readUserD1StorageBytes` plus bucket estimates); that path does **not** +reserve bytes in D1 or UserMeter. + **Strong enforcement:** `consumeDailyEntitlement` and inbound `consumeInboundDelivery` RPCs check the plan limit and increment inside the DO. The Durable Object request model serializes mutations per user; counter updates @@ -167,15 +178,76 @@ mirror is scheduled from the email path via `scheduleAbsoluteDailyEntitlementMirror` so the email subsystem does not duplicate mirror SQL. -**Remaining migration dependency:** retiring the D1 mirror and -`entitlement_daily_counters` table waits until reporting-off-D1 work merges and -mirror parity is verified in production. This expand phase intentionally does -**not** move `storage_bytes` enforcement, `package_service_states` running -counts, or account-deletion write fencing to UserMeter. +### D1 payload storage bytes — UserMeter shadow (expand phase slice 3) + +D1 `users.d1_storage_bytes` and `users.d1_storage_bytes_updated_at` remain the +**sole read, enforcement, and reconciliation authority** for the D1 payload +component of `storage_bytes`. Every caller — including email — reserves through +the same conditional D1 `UPDATE` in `assertWithinStorageBytesEntitlement` +(bounded retry while a real users row remains under limit; fail closed if +contention never resolves; missing-user synthetic contexts keep prior +under-limit allow / over-limit deny). Denial semantics and +`EntitlementLimitError` always come from D1. + +UserMeter schema **v4** adds a singleton `storage_bytes_state` row (`id = 1`) as +**best-effort shadow / future-cutover support only**. The DO row is never read +for usage display or enforcement in this slice. + +**Optional non-email shadow writes:** DO-backed write chokepoints (values, +secrets, memories, and saved-package projections) may pass optional `env` into +`assertWithinStorageBytesEntitlement`. After a **successful D1 reserve**, the +service schedules a non-awaited best-effort shadow via `waitUntil` when +available: re-read the latest D1 value, then absolute-set UserMeter through +`setStorageBytes`. Shadow tasks re-read D1 at execution time so a delayed shadow +cannot apply a stale absolute and leave the DO behind D1. Missing `USER_METER` +or shadow failures log `entitlement-storage-bytes-shadow-failed` and never +affect D1 enforcement. + +**Optional reconcile shadow:** `reconcileUserD1StorageBytes` absolute-sets D1 +from `calculateUserD1StorageBytes` and awaits that write. When optional `env` is +present, it best-effort shadows the post-update D1 value into UserMeter; shadow +failures are logged and never fail or poison the lane. Current +`d1_storage_reconciliation` wiring passes `{ db }` only. + +**Usage reads stay on D1:** account usage, admin drill-down, and +`readCurrentEntitlementResourceUsage` for `storage_bytes` call +`readUserD1StorageBytes` only. + +**Account export and purge:** `UserMeter.exportCounters` may return additive +non-authoritative `storageBytesShadow` on the first page only (`startAfter` +absent) when the shadow row exists; subsequent pages return `null` so paged +consumers never double-count it (still counted once in the `user_meter` section +total). `UserMeter.purge()` clears counters, inbound delivery claims, and any +shadow storage state via `deleteAll`. + +### Future storage authority flip (contract follow-up) + +A separate contract PR — not a merge blocker for this additive slice — will flip +D1 payload storage-byte authority into UserMeter once: + +1. **mailbox-do** passes `env` on email inbound/outbound storage reservations, + and +2. **cron-restructure** passes `env` into `d1_storage_reconciliation` lane + wiring in `index.ts`. + +Only then do reads, reserves, and reconciliation switch to UserMeter-first with +D1 as mirror/cursor. Until that flip, shadow divergence is acceptable; D1 +remains the contract. + +**Remaining UserMeter expand roadmap:** slice 4 — `package_service_states` +running counts (service liveness); slice 5 — account-deletion write fencing. +Storage authority flip is tracked separately as the contract follow-up above. + +**Daily-counter mirror retirement:** dropping D1 `entitlement_daily_counters` +waits until reporting-off-D1 work merges and mirror parity is verified in +production. Module wiring: `consumeDailyEntitlement`, `refundDailyEntitlement`, and -`readDailyEntitlementResourceUsage` in `service.ts` require `env.USER_METER` and -fail closed when the binding is missing. +`readDailyEntitlementResourceUsage` require `env.USER_METER` and fail closed +when the binding is missing. Storage-byte helpers (`readUserD1StorageBytes`, +`assertWithinStorageBytesEntitlement`, `reconcileUserD1StorageBytes`) treat +`USER_METER` as optional shadow-only; missing binding is a no-op for shadow +scheduling. ## Schema history @@ -351,60 +423,65 @@ Rules: against the limit instead of an accumulating count: the enforcement point passes the candidate size via `getCurrent` with `requested: 0`. There is no built-in counter for these. -- **Storage-byte limits** (`storage_bytes`) store the D1 payload estimate on - `users.d1_storage_bytes` (migration 0122). Entitlement checks read that - indexed user row and D1 write chokepoints atomically reserve their existing - positive byte-delta estimate before the write, so mailbox growth and other hot - writes never rescan the user's accumulated rows. The estimate covers - user-owned rows with durable payloads (`email_messages.raw_size` plus - extracted message bodies/metadata, externally stored attachments, values, - encrypted secrets, memories, saved-package projections, jobs, repo/session - metadata, package invocation results, and published artifact metadata). Run - records in the per-user `RunLog` Durable Object are intentionally **excluded** - from the quota — they are observability history, not user content. A fixed - batch of eight oldest users is reconciled every five-minute cron tick by - `d1_storage_reconciliation`; each selected counter is replaced with the - authoritative cross-surface sum. This bounded lane corrects over-reservation - after failed writes, shrinkage from deletes, and writes on surfaces without a - storage-byte chokepoint. Failed rows retain their conservative value and - rotate to the back of the queue for a later retry. StorageRunner Durable - Object buckets expose their own `estimatedBytes`, and each bucket's latest - measurement is persisted on its `user_storage_buckets.estimated_bytes` - inventory row (migration 0118). Write chokepoints pass `getCurrent` as - `D1 counter + sum of per-bucket estimates`, where only the bucket that - triggers the baseline read (plus any inventoried bucket with no stored - estimate yet) is probed live; every other bucket contributes its stored D1 - estimate, so mutating writes do not fan `getEstimatedBytes` RPCs across the - user's whole bucket inventory. Live probe results are persisted - fire-and-forget with **UPDATE-only** statements (they can never recreate an - inventory row removed by account, package, or job deletion), and mutating - StorageRunner RPCs opportunistically refresh their own bucket's stored - estimate after the write, throttled per bucket per isolate - (`storageBucketEstimateRefreshMinIntervalMs`). Stored estimates are therefore - freshness hints with bounded lag — acceptable for an order-of-magnitude cap - because the bucket paying the baseline read is measured live and the run cache - accounts for the run's own accepted writes. `requested` is the candidate - payload size when known. Pure read-only `storage.sql` / `packageStorage().sql` - statements (`SELECT` / `EXPLAIN` / schema `PRAGMA`) skip the baseline read - entirely even when the helper marks the call writable. Mutating SQL and - `storage.set` in one sandbox share a per-run baseline cache so repeated writes - do not re-read the baseline; a later write in the same run that targets a - **different** already-inventoried bucket reuses that bucket's stored estimate - rather than probing it live (bounded staleness, same trade-off as peers). Each - live estimate read waits at most ~2s via `Promise.race` and is retried with - backoff (`storageEstimateReadRetryDelaysMs`; a single 150ms retry lost to - transient per-bucket DO read failures in production) before failing closed for - the caller; the underlying DO RPC is not cancelled if the runtime keeps it - running. A cron lane (`storage_bucket_estimate_backfill`, - `packages/worker/src/storage-buckets/estimate-backfill.ts`) sweeps inventory - rows without a stored estimate in bounded batches (failed probes stay - unmeasured and are retried on later sweeps) so freshly migrated inventories - converge to stored estimates within a few ticks instead of making each user's - first mutating write pay (and possibly fail on) the whole-inventory probe. The - counter intentionally does **not** attempt to scan Cloudflare Artifacts - repository contents, KV snapshot/bundle bodies, R2 object listings beyond - `email_messages.raw_size`, or Vectorize: those stores either lack reliable - byte metadata or are derived from D1 and are documented in `data-storage.md`. +- **Storage-byte limits** (`storage_bytes`) split into two quota components: + 1. **D1 payload bytes (authoritative on D1):** user-owned D1 rows with durable + payloads (`email_messages.raw_size` plus extracted message bodies/metadata, + externally stored attachments, values, encrypted secrets, memories, + saved-package projections, jobs, repo/session metadata, package invocation + results, and published artifact metadata). Run records in the per-user + `RunLog` Durable Object are intentionally **excluded** — they are + observability history, not user content. Write chokepoints atomically + reserve positive byte deltas via `assertWithinStorageBytesEntitlement` + against `users.d1_storage_bytes` (every caller, including email). Optional + `env` on non-email paths schedules a best-effort UserMeter shadow after a + successful D1 reserve; shadow state is not read for enforcement or usage. + The bounded `d1_storage_reconciliation` lane recomputes the authoritative + cross-surface sum via `calculateUserD1StorageBytes` and absolute-sets D1; + optional `env` shadows into UserMeter after the D1 write — see + [UserMeter (expand phase)](#usermeter-expand-phase). + + 2. **StorageRunner bucket estimates (separate):** each bucket exposes + `estimatedBytes`; inventory rows persist the latest measurement on + `user_storage_buckets.estimated_bytes` (migration 0118). Write chokepoints + that pass `getCurrent` compose + `D1 payload (readUserD1StorageBytes) + sum of per-bucket estimates` for a + **check-only** entitlement comparison — `getCurrent` never reserves in D1 + or UserMeter. Only the bucket that triggers the baseline read (plus any + inventoried bucket with no stored estimate yet) is probed live; every other + bucket contributes its stored D1 estimate, so mutating writes do not fan + `getEstimatedBytes` RPCs across the whole inventory. Live probe results are + persisted fire-and-forget with **UPDATE-only** statements (they can never + recreate an inventory row removed by account, package, or job deletion), + and mutating StorageRunner RPCs opportunistically refresh their own + bucket's stored estimate after the write, throttled per bucket per isolate + (`storageBucketEstimateRefreshMinIntervalMs`). Stored estimates are + freshness hints with bounded lag — acceptable for an order-of-magnitude cap + because the bucket paying the baseline read is measured live and the run + cache accounts for the run's own accepted writes. `requested` is the + candidate payload size when known. Pure read-only `storage.sql` / + `packageStorage().sql` statements (`SELECT` / `EXPLAIN` / schema `PRAGMA`) + skip the baseline read entirely even when the helper marks the call + writable. Mutating SQL and `storage.set` in one sandbox share a per-run + baseline cache so repeated writes do not re-read the baseline; a later + write in the same run that targets a **different** already-inventoried + bucket reuses that bucket's stored estimate rather than probing it live + (bounded staleness, same trade-off as peers). Each live estimate read waits + at most ~2s via `Promise.race` and is retried with backoff + (`storageEstimateReadRetryDelaysMs`; a single 150ms retry lost to transient + per-bucket DO read failures in production) before failing closed for the + caller; the underlying DO RPC is not cancelled if the runtime keeps it + running. A cron lane (`storage_bucket_estimate_backfill`, + `packages/worker/src/storage-buckets/estimate-backfill.ts`) sweeps + inventory rows without a stored estimate in bounded batches (failed probes + stay unmeasured and are retried on later sweeps) so freshly migrated + inventories converge to stored estimates within a few ticks instead of + making each user's first mutating write pay (and possibly fail on) the + whole-inventory probe. The D1 payload counter intentionally does **not** + attempt to scan Cloudflare Artifacts repository contents, KV + snapshot/bundle bodies, R2 object listings beyond + `email_messages.raw_size`, or Vectorize: those stores either lack reliable + byte metadata or are derived from D1 and are documented in + `data-storage.md`. ### Concurrency @@ -458,20 +535,20 @@ Use `getCurrent` only when the built-in D1 counter cannot express the resource. ## Enforcement points -| Resource | Enforcement point | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `scheduled_jobs` | `createJob` in `packages/worker/src/jobs/service.ts` (exemplar) | -| `saved_packages` | new-package branch of `package_save` and projection insert | -| `package_services` | `service_start` capability path (count from `package_service_states`) | -| `persistent_package_services` | `service_start` for services declared `mode: 'persistent'` | -| `repo_sessions` | `repo_open_session` before creating a new session | -| `email_sends_per_day` | `sendOutboundEmail` (`consumeDailyEntitlement`; plan limit from `resolvePlanLimit`) | -| `email_receives_per_day` | `handleInboundEmail` (`consumeDailyEntitlement`; same plan limits; refund only on `RetryableInboundStorageError`) | -| `stored_email_messages` | `handleInboundEmail` before storage (`assertWithinEntitlement`; `max` caps from `planLimits.max`) | -| `email_message_bytes` | `handleInboundEmail` before quota/parse (per-message raw size via `resolveEmailResourceLimit`) | -| `secrets` | new-entry branch of `saveSecret` in `packages/worker/src/mcp/secrets/service.ts` | -| `concurrent_workflows` | `createDynamicCallableWorkflow` (`reserveWorkflowProjectionSlot` + `assertWithinEntitlement` getCurrent; `max` = 5,000) | -| `storage_bytes` | D1 payload writes (values, secrets, memories, saved-package projections, email storage) and StorageRunner write tools/app RPCs | +| Resource | Enforcement point | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `scheduled_jobs` | `createJob` in `packages/worker/src/jobs/service.ts` (exemplar) | +| `saved_packages` | new-package branch of `package_save` and projection insert | +| `package_services` | `service_start` capability path (count from `package_service_states`) | +| `persistent_package_services` | `service_start` for services declared `mode: 'persistent'` | +| `repo_sessions` | `repo_open_session` before creating a new session | +| `email_sends_per_day` | `sendOutboundEmail` (`consumeDailyEntitlement`; plan limit from `resolvePlanLimit`) | +| `email_receives_per_day` | `handleInboundEmail` (`consumeDailyEntitlement`; same plan limits; refund only on `RetryableInboundStorageError`) | +| `stored_email_messages` | `handleInboundEmail` before storage (`assertWithinEntitlement`; `max` caps from `planLimits.max`) | +| `email_message_bytes` | `handleInboundEmail` before quota/parse (per-message raw size via `resolveEmailResourceLimit`) | +| `secrets` | new-entry branch of `saveSecret` in `packages/worker/src/mcp/secrets/service.ts` | +| `concurrent_workflows` | `createDynamicCallableWorkflow` (`reserveWorkflowProjectionSlot` + `assertWithinEntitlement` getCurrent; `max` = 5,000) | +| `storage_bytes` | D1 payload writes via `assertWithinStorageBytesEntitlement` (D1 reserve for all callers; optional `env` shadow on non-email paths) and StorageRunner write tools/app RPCs (`getCurrent` check-only for bucket component) | ## Billing diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index a86960099b..5302603cd6 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -513,8 +513,8 @@ primitives: group: storage name: User meter summary: - Per-user UserMeter DO SQLite for daily entitlement counters and inbound - delivery idempotency. + Per-user UserMeter DO SQLite for daily entitlement counters, inbound + delivery idempotency, and expand-phase D1 storage-byte shadow. code: - packages/worker/src/entitlements/user-meter-do.ts - packages/worker/src/entitlements/user-meter-client.ts diff --git a/packages/worker/src/account/export.node.test.ts b/packages/worker/src/account/export.node.test.ts index 08e9964d21..815d16f8cb 100644 --- a/packages/worker/src/account/export.node.test.ts +++ b/packages/worker/src/account/export.node.test.ts @@ -1572,6 +1572,12 @@ test('account export includes user_meter counters, pages them, and warns on trun mirrorUpdatedAt: 'r/00000000000000000001', }, ] + const storageBytesShadow = { + bytes: 4_096, + revision: 3, + updatedAt: '2026-07-31T03:00:00.000Z', + mirrorUpdatedAt: 'r/00000000000000000003', + } const exportCounters = vi.fn( async (input: { pageSize?: number; startAfter?: string | null }) => { const pageSize = input.pageSize ?? 100 @@ -1582,8 +1588,11 @@ test('account export includes user_meter counters, pages them, and warns on trun : 0 const page = counters.slice(startIndex, startIndex + pageSize) const truncated = startIndex + pageSize < counters.length + const isFirstPage = + typeof input.startAfter !== 'string' || input.startAfter.length === 0 return { counters: page, + storageBytesShadow: isFirstPage ? storageBytesShadow : null, nextStartAfter: truncated ? `${page.at(-1)!.day}:${page.at(-1)!.resource}` : null, @@ -1606,9 +1615,10 @@ test('account export includes user_meter counters, pages them, and warns on trun mcpUserId: 'user-aaa', }) expect(idFromName).toHaveBeenCalledWith('user-aaa') - expect(accountExport.manifest.sections.user_meter?.count).toBe(3) + expect(accountExport.manifest.sections.user_meter?.count).toBe(4) expect(accountExport.durableObjects.userMeter).toEqual({ counters, + storageBytesShadow, nextStartAfter: null, truncated: false, }) @@ -1626,6 +1636,7 @@ test('account export includes user_meter counters, pages them, and warns on trun pageSize: 2, }) expect(first.items).toEqual(counters.slice(0, 2)) + expect(first.storageBytesShadow).toEqual(storageBytesShadow) expect(first.truncated).toBe(true) expect(first.nextStartAfter).toBe('2026-07-30:execute_calls_per_day') @@ -1638,12 +1649,19 @@ test('account export includes user_meter counters, pages them, and warns on trun startAfter: first.nextStartAfter ?? undefined, }) expect(second.items).toEqual(counters.slice(2)) + expect(second.storageBytesShadow).toBeNull() expect(second.truncated).toBe(false) expect(second.nextStartAfter).toBeNull() - expect(exportCounters).toHaveBeenCalled() + expect(exportCounters).toHaveBeenCalledWith( + expect.objectContaining({ + pageSize: 2, + startAfter: first.nextStartAfter, + }), + ) exportCounters.mockImplementation(async () => ({ counters: [counters[0]!], + storageBytesShadow: null, nextStartAfter: 'cursor-more', truncated: true, })) @@ -1653,6 +1671,7 @@ test('account export includes user_meter counters, pages them, and warns on trun mcpUserId: 'user-aaa', }) expect(truncatedExport.durableObjects.userMeter?.truncated).toBe(true) + expect(truncatedExport.manifest.sections.user_meter?.count).toBe(1) expect(truncatedExport.manifest.warnings).toContain( 'User meter counters were truncated in the full export; use account_export_section with section "user_meter" to retrieve additional pages.', ) diff --git a/packages/worker/src/account/export.ts b/packages/worker/src/account/export.ts index c3bf88a67e..a2ac581228 100644 --- a/packages/worker/src/account/export.ts +++ b/packages/worker/src/account/export.ts @@ -204,6 +204,10 @@ export type AccountExportArtifactRepo = { type RunRecordsExportPayload = Awaited> +function countUserMeterExportEntries(result: UserMeterExportResult): number { + return result.counters.length + (result.storageBytesShadow == null ? 0 : 1) +} + function countRunRecordsExportEntries( runRecords: RunRecordsExportPayload | null | undefined, ) { @@ -279,6 +283,11 @@ export type AccountExportSectionResult = { nextStartAfter: string | null pageSize: number warnings: Array + /** + * Non-authoritative UserMeter storage-byte shadow. Present only on the + * first `user_meter` page (`startAfter` absent); later pages omit it. + */ + storageBytesShadow?: UserMeterExportResult['storageBytesShadow'] } function normalizePageSize(pageSize: number | undefined) { @@ -1050,7 +1059,7 @@ async function collectManifestInventoryCounts(input: { }).exportCounters({ pageSize: maxExportPageSize, }) - return page.counters.length + return countUserMeterExportEntries(page) }), safeCount('remote connectors', async () => countScalar( @@ -1571,7 +1580,7 @@ function buildManifest(input: { count: input.durableObjects?.userMeter == null ? (input.inventoryCounts?.userMeterCounters ?? 0) - : input.durableObjects.userMeter.counters.length, + : countUserMeterExportEntries(input.durableObjects.userMeter), warnings: input.warnings.filter( (warning) => warning.startsWith('User meter ') || warning.startsWith('USER_METER '), @@ -1911,6 +1920,7 @@ export async function readAccountExportSection(input: { return { section: input.section, items: page.counters, + storageBytesShadow: page.storageBytesShadow, truncated: page.truncated, nextStartAfter: page.nextStartAfter, pageSize, diff --git a/packages/worker/src/account/user-owned-surfaces.ts b/packages/worker/src/account/user-owned-surfaces.ts index 34d779fea8..47a46d6fc3 100644 --- a/packages/worker/src/account/user-owned-surfaces.ts +++ b/packages/worker/src/account/user-owned-surfaces.ts @@ -130,7 +130,7 @@ export const accountUserOwnedDurableObjectSurfaces: ReadonlyArray + d1StorageBytes?: number }) { const stableUserId = testStableUserIdFromEmail(input.email) const dailyCounters = input.dailyCounters?.map((row) => ({ ...row })) ?? [] + const d1StorageBytes = input.d1StorageBytes ?? 0 return { stableUserId, db: { @@ -60,10 +62,12 @@ function createUsageTestDb(input: { ) return (row ? { count: row.count } : { count: 0 }) as T } + if (normalized.includes('d1_storage_bytes')) { + return { bytes: d1StorageBytes } as T + } if ( normalized.includes('count(*)') || - normalized.includes('sum(') || - normalized.includes('d1_storage_bytes') + normalized.includes('sum(') ) { return { count: 0, total: 0, bytes: 0 } as T } @@ -216,3 +220,24 @@ test('loadAccountUsageData returns plan rows and authoritative UserMeter daily c expect(currentFor(authoritative, 'concurrent_workflows')).toBe(3) expect(currentFor(authoritative, 'saved_packages')).toBe(4) }) + +test('loadAccountUsageData prefers D1 storage_bytes over a divergent UserMeter shadow', async () => { + const now = new Date('2026-07-25T12:00:00.000Z') + const email = 'usage-storage@example.com' + const userId = testStableUserIdFromEmail(email) + const { db } = createUsageTestDb({ + userId: 11, + email, + plan: 'pro', + packageCount: 0, + d1StorageBytes: 4_321, + }) + const usageEnv = withUsageEnv({ APP_DB: db }) + await usageEnv.meter.seedStorageBytes({ userId, bytes: 321 }) + const data = await loadAccountUsageData({ + env: usageEnv as Env, + userId: 11, + now, + }) + expect(currentFor(data, 'storage_bytes')).toBe(4_321) +}) diff --git a/packages/worker/src/entitlements/d1-storage-reconciliation.ts b/packages/worker/src/entitlements/d1-storage-reconciliation.ts index 7428ed8496..0759de7b6c 100644 --- a/packages/worker/src/entitlements/d1-storage-reconciliation.ts +++ b/packages/worker/src/entitlements/d1-storage-reconciliation.ts @@ -1,3 +1,4 @@ +import { type UserMeterEnv } from './user-meter-client.ts' import { listUsersForD1StorageReconciliation, markUserD1StorageReconciliationAttempt, @@ -9,6 +10,8 @@ import { * Reservations keep the hot path conservative between sweeps; this lane * corrects drift from deletes, failed writes after reservation, and D1 * surfaces whose writes do not pass through storage-byte enforcement. + * Optional `env` best-effort shadows each absolute into UserMeter after the + * D1 update; shadow failures never fail or poison the lane. */ export const d1StorageReconciliationBatchSize = 8 @@ -16,6 +19,8 @@ export async function reconcileD1StorageBytes(input: { db: D1Database now?: Date batchSize?: number + /** Optional expand-phase UserMeter shadow after each D1 absolute update. */ + env?: UserMeterEnv }): Promise<{ scanned: number; updated: number; failed: number }> { const now = input.now ?? new Date() const rows = await listUsersForD1StorageReconciliation({ @@ -28,6 +33,7 @@ export async function reconcileD1StorageBytes(input: { try { const result = await reconcileUserD1StorageBytes({ db: input.db, + env: input.env, userId: row.userId, now, }) diff --git a/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts b/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts index d9daeacd68..a690d38491 100644 --- a/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts +++ b/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts @@ -1,11 +1,13 @@ import { env } from 'cloudflare:test' import { expect, test } from 'vitest' import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' import { calculateUserD1StorageBytes, readUserD1StorageBytes, } from './service.ts' import { reconcileD1StorageBytes } from './d1-storage-reconciliation.ts' +import { userMeterRpc } from './user-meter-client.ts' async function insertReconciliationUser(input: { userId: string @@ -29,7 +31,7 @@ async function insertReconciliationUser(input: { .run() } -test('D1 storage reconciliation is bounded and replaces stored drift with authoritative bytes', async () => { +test('D1 storage reconciliation is bounded and remains D1-authoritative with {db}', async () => { await ensureEmailTestSchema(env.APP_DB) await env.APP_DB.prepare( `UPDATE users @@ -71,6 +73,11 @@ test('D1 storage reconciliation is bounded and replaces stored drift with author }) expect(expectedFirstBytes).toBeGreaterThan(128) + const firstMeter = userMeterRpc({ env, userId: firstUserId }) + expect(await firstMeter.readStorageBytes()).toEqual({ + outcome: 'needs_bootstrap', + }) + await expect( reconcileD1StorageBytes({ db: env.APP_DB, @@ -78,9 +85,13 @@ test('D1 storage reconciliation is bounded and replaces stored drift with author batchSize: 1, }), ).resolves.toEqual({ scanned: 1, updated: 1, failed: 0 }) + await expect( readUserD1StorageBytes({ db: env.APP_DB, userId: firstUserId }), ).resolves.toBe(expectedFirstBytes) + expect(await firstMeter.readStorageBytes()).toEqual({ + outcome: 'needs_bootstrap', + }) await expect( readUserD1StorageBytes({ db: env.APP_DB, userId: secondUserId }), ).resolves.toBe(99) @@ -96,3 +107,104 @@ test('D1 storage reconciliation is bounded and replaces stored drift with author readUserD1StorageBytes({ db: env.APP_DB, userId: secondUserId }), ).resolves.toBe(0) }) + +test('optional reconciliation env shadows UserMeter without becoming authoritative', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = '3'.repeat(64) + await insertReconciliationUser({ + userId, + email: `${crypto.randomUUID()}@example.test`, + storedBytes: 50, + updatedAt: '2019-01-01T00:00:00.000Z', + }) + await env.APP_DB.prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, text_body, raw_size, + processing_status, created_at, updated_at + ) VALUES (?, 'inbound', ?, 'sender@example.test', 'subject', 'body', 200, + 'stored', ?, ?)`, + ) + .bind( + crypto.randomUUID(), + userId, + '2026-07-31T00:00:00.000Z', + '2026-07-31T00:00:00.000Z', + ) + .run() + + const meter = userMeterRpc({ env, userId }) + await meter.setStorageBytes({ + bytes: 999_999, + updatedAt: '2026-07-31T00:00:00.000Z', + }) + + const expectedBytes = await calculateUserD1StorageBytes({ + db: env.APP_DB, + userId, + }) + await expect( + reconcileD1StorageBytes({ + db: env.APP_DB, + env, + now: new Date('2026-07-31T01:00:00.000Z'), + batchSize: 8, + }), + ).resolves.toMatchObject({ failed: 0 }) + + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId }), + ).resolves.toBe(expectedBytes) + expect(await meter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: expectedBytes, + }) +}) + +test('reconciliation shadow failure does not fail or poison the D1 lane', async () => { + await ensureEmailTestSchema(env.APP_DB) + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes_updated_at = '9999-12-31T23:59:59.999Z'`, + ).run() + const userId = '4'.repeat(64) + await insertReconciliationUser({ + userId, + email: `${crypto.randomUUID()}@example.test`, + storedBytes: 40, + updatedAt: '2018-01-01T00:00:00.000Z', + }) + + consoleWarn.mockImplementation(() => {}) + const failingMeterEnv = { + USER_METER: { + idFromName: env.USER_METER.idFromName.bind(env.USER_METER), + get() { + return { + async setStorageBytes() { + throw new Error('shadow write failed') + }, + } + }, + }, + } as unknown as typeof env + + const expectedBytes = await calculateUserD1StorageBytes({ + db: env.APP_DB, + userId, + }) + await expect( + reconcileD1StorageBytes({ + db: env.APP_DB, + env: failingMeterEnv, + now: new Date('2026-07-31T01:00:00.000Z'), + batchSize: 1, + }), + ).resolves.toEqual({ scanned: 1, updated: 1, failed: 0 }) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId }), + ).resolves.toBe(expectedBytes) + expect(consoleWarn).toHaveBeenCalledWith( + 'entitlement-storage-bytes-shadow-failed', + expect.any(Error), + ) +}) diff --git a/packages/worker/src/entitlements/entitlements.node.test.ts b/packages/worker/src/entitlements/entitlements.node.test.ts index 40d5ee125d..f1aab23582 100644 --- a/packages/worker/src/entitlements/entitlements.node.test.ts +++ b/packages/worker/src/entitlements/entitlements.node.test.ts @@ -1143,6 +1143,110 @@ test('storage bytes enforce for planned users and enforce finite max storage cap ).toBe(true) }) +test('storage byte reserve retries after a failed conditional update under limit', async () => { + const userId = await createStableUserIdFromEmail(plannedEmail) + const proLimit = planLimits.pro.maxStorageBytes + let reserveAttempts = 0 + let bytes = proLimit - 10 + const db = { + prepare(query: string) { + return { + bind(...params: Array) { + return { + async first() { + if ( + query.includes('SELECT plan, stripe_plan FROM users') || + query.includes('SELECT plan FROM users') + ) { + return { plan: 'pro', stripe_plan: null } as T + } + if (query.includes('SELECT d1_storage_bytes AS bytes')) { + return { bytes } as T + } + throw new Error(`Unsupported first query: ${query}`) + }, + async run() { + if ( + query.includes('SET d1_storage_bytes = d1_storage_bytes + ?') + ) { + reserveAttempts += 1 + if (reserveAttempts === 1) { + return { meta: { changes: 0 } } + } + const requested = Number(params[0]) + if (bytes + Number(params[3]) > Number(params[4])) { + return { meta: { changes: 0 } } + } + bytes += requested + return { meta: { changes: 1 } } + } + throw new Error(`Unsupported run query: ${query}`) + }, + } + }, + } + }, + } as unknown as D1Database + + await assertWithinStorageBytesEntitlement({ + db, + userId, + email: plannedEmail, + requested: 5, + }) + expect(reserveAttempts).toBe(2) + expect(bytes).toBe(proLimit - 5) +}) + +test('storage byte reserve fails closed when contention never resolves', async () => { + const userId = await createStableUserIdFromEmail(plannedEmail) + const proLimit = planLimits.pro.maxStorageBytes + let reserveAttempts = 0 + const db = { + prepare(query: string) { + return { + bind() { + return { + async first() { + if ( + query.includes('SELECT plan, stripe_plan FROM users') || + query.includes('SELECT plan FROM users') + ) { + return { plan: 'pro', stripe_plan: null } as T + } + if (query.includes('SELECT d1_storage_bytes AS bytes')) { + return { bytes: proLimit - 10 } as T + } + throw new Error(`Unsupported first query: ${query}`) + }, + async run() { + if ( + query.includes('SET d1_storage_bytes = d1_storage_bytes + ?') + ) { + reserveAttempts += 1 + return { meta: { changes: 0 } } + } + throw new Error(`Unsupported run query: ${query}`) + }, + } + }, + } + }, + } as unknown as D1Database + + await expect( + assertWithinStorageBytesEntitlement({ + db, + userId, + email: plannedEmail, + requested: 5, + }), + ).rejects.toThrow( + 'Storage byte reservation could not complete under concurrent updates.', + ) + expect(reserveAttempts).toBe(5) +}) + test('getPlanRank orders free < pro < partner < max', () => { expect(getPlanRank('free')).toBeLessThan(getPlanRank('pro')) expect(getPlanRank('pro')).toBeLessThan(getPlanRank('partner')) diff --git a/packages/worker/src/entitlements/service.ts b/packages/worker/src/entitlements/service.ts index 93d8e72435..5d3a5e6ae5 100644 --- a/packages/worker/src/entitlements/service.ts +++ b/packages/worker/src/entitlements/service.ts @@ -13,7 +13,11 @@ import { isDailyEntitlementResource, type DailyEntitlementResource, } from './user-meter-do.ts' -import { userMeterRpc, type UserMeterEnv } from './user-meter-client.ts' +import { + userMeterNamespace, + userMeterRpc, + type UserMeterEnv, +} from './user-meter-client.ts' /** Env surface for authoritative entitlement usage readers (UserMeter + RunLog). */ export type EntitlementUsageEnv = UserMeterEnv & Pick @@ -682,10 +686,49 @@ export async function calculateUserD1StorageBytes(input: { return sums.reduce((total, value) => total + value, 0) } -export async function readUserD1StorageBytes(input: { +/** Best-effort D1→UserMeter absolute shadow; re-reads D1 at task time. */ +async function shadowUserMeterStorageBytesFromD1(input: { + db: D1Database + env: UserMeterEnv + userId: string +}) { + if (!userMeterNamespace(input.env)) return + const bytes = await readUserD1StorageBytes({ + db: input.db, + userId: input.userId, + }) + const meter = userMeterRpc({ env: input.env, userId: input.userId }) + await meter.setStorageBytes({ + bytes, + updatedAt: new Date().toISOString(), + }) +} + +function scheduleUserMeterStorageBytesShadow(input: { db: D1Database + env: UserMeterEnv | undefined userId: string + waitUntil?: (promise: Promise) => void }) { + if (!input.env || !userMeterNamespace(input.env)) return + const tracked = shadowUserMeterStorageBytesFromD1({ + db: input.db, + env: input.env, + userId: input.userId, + }).catch((error: unknown) => { + console.warn('entitlement-storage-bytes-shadow-failed', error) + }) + if (input.waitUntil) { + input.waitUntil(tracked) + return + } + void tracked +} + +async function readUserD1StorageBytesRow(input: { + db: D1Database + userId: string +}): Promise<{ bytes: number } | null> { const row = await input.db .prepare( `SELECT d1_storage_bytes AS bytes @@ -694,13 +737,29 @@ export async function readUserD1StorageBytes(input: { ) .bind(input.userId) .first<{ bytes: number }>() - return Math.max(0, Number(row?.bytes ?? 0)) + if (row == null) return null + return { bytes: Math.max(0, Number(row.bytes ?? 0)) } } +export async function readUserD1StorageBytes(input: { + db: D1Database + userId: string +}): Promise { + const row = await readUserD1StorageBytesRow(input) + return row?.bytes ?? 0 +} + +/** + * Replace one user's D1 storage-byte counter with the cross-surface D1 payload + * sum. Optional `env` best-effort shadows into UserMeter after the D1 write; + * shadow failures never fail or poison this reconcile. + */ export async function reconcileUserD1StorageBytes(input: { db: D1Database userId: string now?: Date + /** Optional expand-phase UserMeter shadow after the D1 absolute update. */ + env?: UserMeterEnv }) { const bytes = await calculateUserD1StorageBytes(input) const result = await input.db @@ -712,7 +771,19 @@ export async function reconcileUserD1StorageBytes(input: { ) .bind(bytes, (input.now ?? new Date()).toISOString(), input.userId) .run() - return { bytes, updated: (result.meta.changes ?? 0) > 0 } + const updated = (result.meta.changes ?? 0) > 0 + if (input.env) { + try { + await shadowUserMeterStorageBytesFromD1({ + db: input.db, + env: input.env, + userId: input.userId, + }) + } catch (error) { + console.warn('entitlement-storage-bytes-shadow-failed', error) + } + } + return { bytes, updated } } export async function listUsersForD1StorageReconciliation(input: { @@ -923,12 +994,31 @@ export async function readCurrentEntitlementResourceUsage(input: { }) } +/** Bounded retries when a conditional D1 reserve loses to concurrent writers. */ +const storageBytesReserveMaxAttempts = 5 + +/** + * Atomically reserve D1 payload storage bytes against the plan limit. + * Optional `env` schedules a best-effort UserMeter shadow after a successful + * reserve (never awaited; failures logged only). `getCurrent` is check-only + * and does not reserve in D1 or UserMeter. + * + * Real users never succeed unreserved: a failed conditional update re-reads + * row existence/current, throws {@link EntitlementLimitError} at the limit, + * retries while a real row remains under limit, and fails closed with an + * operational error if contention never resolves. Missing-user synthetic + * contexts keep their prior under-limit allow / over-limit deny semantics. + */ export async function assertWithinStorageBytesEntitlement(input: { db: D1Database userId: string email: string | null | undefined requested?: number getCurrent?: () => Promise + /** Optional expand-phase UserMeter shadow after a successful D1 reserve. */ + env?: UserMeterEnv + /** Prefer `ctx.waitUntil` for the UserMeter shadow; never awaited here. */ + waitUntil?: (promise: Promise) => void }) { if (input.getCurrent) { await assertWithinEntitlement({ @@ -948,34 +1038,66 @@ export async function assertWithinStorageBytesEntitlement(input: { }) const limit = resolvePlanLimit(plan, 'storage_bytes') const requested = Math.max(0, input.requested ?? 1) - const now = new Date().toISOString() - const result = await input.db - .prepare( - `UPDATE users - SET d1_storage_bytes = d1_storage_bytes + ?, - d1_storage_bytes_updated_at = ? - WHERE stable_user_id = ? - AND d1_storage_bytes + ? <= ?`, - ) - .bind(requested, now, input.userId, requested, limit) - .run() - if ((result.meta.changes ?? 0) > 0) return - const current = await readUserD1StorageBytes({ - db: input.db, - userId: input.userId, - }) - // Synthetic contexts have no users row to reserve against. Their plan has - // already failed closed to `free`; preserve that finite limit without - // inventing a user-owned counter row that reconciliation cannot resolve. - if (current + requested <= limit) return - throw new EntitlementLimitError({ - resource: 'storage_bytes', - plan, - limit, - current, - upgradeHint: buildEntitlementUpgradeHint('storage_bytes'), - }) + for (let attempt = 0; attempt < storageBytesReserveMaxAttempts; attempt++) { + const now = new Date().toISOString() + const result = await input.db + .prepare( + `UPDATE users + SET d1_storage_bytes = d1_storage_bytes + ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ? + AND d1_storage_bytes + ? <= ?`, + ) + .bind(requested, now, input.userId, requested, limit) + .run() + if ((result.meta.changes ?? 0) > 0) { + scheduleUserMeterStorageBytesShadow({ + db: input.db, + env: input.env, + userId: input.userId, + waitUntil: input.waitUntil, + }) + return + } + + const row = await readUserD1StorageBytesRow({ + db: input.db, + userId: input.userId, + }) + // Synthetic contexts have no users row to reserve against. Their plan + // has already failed closed to `free`; preserve that finite limit + // without inventing a user-owned counter row that reconciliation + // cannot resolve. + if (row == null) { + const current = 0 + if (current + requested <= limit) return + throw new EntitlementLimitError({ + resource: 'storage_bytes', + plan, + limit, + current, + upgradeHint: buildEntitlementUpgradeHint('storage_bytes'), + }) + } + + const current = row.bytes + if (current + requested > limit) { + throw new EntitlementLimitError({ + resource: 'storage_bytes', + plan, + limit, + current, + upgradeHint: buildEntitlementUpgradeHint('storage_bytes'), + }) + } + // Real row still under limit: another writer raced the conditional + // UPDATE. Retry the reserve rather than succeeding unreserved. + } + + throw new Error( + 'Storage byte reservation could not complete under concurrent updates.', + ) } export type AssertWithinEntitlementInput = { diff --git a/packages/worker/src/entitlements/user-meter-do.ts b/packages/worker/src/entitlements/user-meter-do.ts index 8616078585..8633bf0b91 100644 --- a/packages/worker/src/entitlements/user-meter-do.ts +++ b/packages/worker/src/entitlements/user-meter-do.ts @@ -26,7 +26,9 @@ export const userMeterDailyCounterRetentionDays = 7 const metaSchemaVersionKey = 'schema_version' /** Bump when initializeSchema DDL changes; warm objects skip DDL. */ -const userMeterSchemaVersion = 3 +const userMeterSchemaVersion = 4 +/** Singleton row id for expand-phase D1 storage-byte shadow (schema v4). */ +const storageBytesStateRowId = 1 const defaultExportPageSize = 100 const maxExportPageSize = 500 @@ -77,8 +79,45 @@ export type UserMeterInitializeResult = UserMeterReadyState & { created: boolean } +export type UserMeterStorageBytesState = { + bytes: number + revision: number + updatedAt: string + mirrorUpdatedAt: string +} + +export type UserMeterStorageBytesReadyState = { + outcome: 'ready' + bytes: number + revision: number + mirrorUpdatedAt: string +} + +export type UserMeterStorageBytesReadResult = + | UserMeterBootstrapState + | UserMeterStorageBytesReadyState + +export type UserMeterStorageBytesReserveResult = + | UserMeterBootstrapState + | (UserMeterStorageBytesReadyState & { reserved: boolean }) + +export type UserMeterStorageBytesInitializeResult = + UserMeterStorageBytesReadyState & { + created: boolean + } + +export type UserMeterStorageBytesSetResult = UserMeterStorageBytesReadyState & { + created: boolean +} + export type UserMeterExportResult = { counters: Array + /** + * Non-authoritative D1 storage-byte shadow. Emitted only on the first + * export page (`startAfter` absent); subsequent pages return `null` so + * paged consumers never double-count the singleton shadow. + */ + storageBytesShadow: UserMeterStorageBytesState | null nextStartAfter: string | null truncated: boolean } @@ -253,6 +292,15 @@ class UserMeterBase extends DurableObject { `CREATE INDEX IF NOT EXISTS idx_inbound_delivery_claims_day ON inbound_delivery_claims (day)`, ) + // Expand-phase shadow of D1 `users.d1_storage_bytes` (schema v4). + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS storage_bytes_state ( + id INTEGER PRIMARY KEY NOT NULL CHECK (id = 1), + bytes INTEGER NOT NULL, + revision INTEGER NOT NULL, + updated_at TEXT NOT NULL + ) + `) this.ctx.storage.sql.exec( `INSERT INTO user_meter_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, @@ -261,6 +309,41 @@ class UserMeterBase extends DurableObject { ) } + private storageReadyState( + bytes: number, + revision: number, + ): UserMeterStorageBytesReadyState { + const safeBytes = Math.max(0, bytes) + const safeRevision = Math.max(0, revision) + return { + outcome: 'ready', + bytes: safeBytes, + revision: safeRevision, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(safeRevision), + } + } + + private readStorageRow(): { + bytes: number + revision: number + updatedAt: string + } | null { + const row = this.ctx.storage.sql + .exec<{ bytes: number; revision: number; updated_at: string }>( + `SELECT bytes, revision, updated_at + FROM storage_bytes_state + WHERE id = ?`, + storageBytesStateRowId, + ) + .toArray()[0] + if (!row) return null + return { + bytes: Math.max(0, Number(row.bytes ?? 0)), + revision: Math.max(0, Number(row.revision ?? 0)), + updatedAt: String(row.updated_at ?? ''), + } + } + private deleteStaleCounters(now: Date) { const cutoffDay = retentionCutoffDay(now) this.ctx.storage.sql.exec( @@ -552,6 +635,143 @@ class UserMeterBase extends DurableObject { return readyState(row.count, row.revision) } + /** Cutover-support cold seed; expand-phase shadow sync uses {@link setStorageBytes}. */ + async initializeStorageBytes(input: { + bytes: number + updatedAt: string + }): Promise { + const bytes = Math.max(0, Math.trunc(Number(input.bytes) || 0)) + const cursor = this.ctx.storage.sql.exec( + `INSERT INTO storage_bytes_state (id, bytes, revision, updated_at) + VALUES (?, ?, 1, ?) + ON CONFLICT(id) DO NOTHING`, + storageBytesStateRowId, + bytes, + input.updatedAt, + ) + const created = cursor.rowsWritten > 0 + const row = this.readStorageRow() + if (!row) { + throw new Error( + 'UserMeter initializeStorageBytes failed to materialize storage state.', + ) + } + return { ...this.storageReadyState(row.bytes, row.revision), created } + } + + /** Cutover-support / shadow read; expand-phase usage and enforcement use D1. */ + async readStorageBytes(): Promise { + const row = this.readStorageRow() + if (!row) return { outcome: 'needs_bootstrap' } + return this.storageReadyState(row.bytes, row.revision) + } + + /** + * Cutover-support atomic reserve. Expand-phase enforcement uses D1 only; + * missing state returns `needs_bootstrap`. + */ + async reserveStorageBytes(input: { + requested: number + limit: number + updatedAt: string + }): Promise { + const requested = Math.max(0, Math.trunc(Number(input.requested) || 0)) + const existing = this.readStorageRow() + if (!existing) { + return { outcome: 'needs_bootstrap' } + } + if ( + requested > 0 && + (input.limit < 1 || existing.bytes + requested > input.limit) + ) { + return { + ...this.storageReadyState(existing.bytes, existing.revision), + reserved: false, + } + } + if (requested === 0) { + return { + ...this.storageReadyState(existing.bytes, existing.revision), + reserved: true, + } + } + const nextBytes = existing.bytes + requested + const nextRevision = existing.revision + 1 + this.ctx.storage.sql.exec( + `UPDATE storage_bytes_state + SET bytes = ?, + revision = ?, + updated_at = ? + WHERE id = ? AND revision = ?`, + nextBytes, + nextRevision, + input.updatedAt, + storageBytesStateRowId, + existing.revision, + ) + const row = this.readStorageRow() + if (!row) { + throw new Error('UserMeter reserveStorageBytes lost storage state.') + } + return { + ...this.storageReadyState(row.bytes, row.revision), + reserved: row.bytes === nextBytes && row.revision === nextRevision, + } + } + + /** + * Absolute shadow-set from D1. Materializes the singleton and bumps + * revision. Callers should re-read latest D1 before invoking so delayed + * shadows cannot leave the DO behind D1. + */ + async setStorageBytes(input: { + bytes: number + updatedAt: string + }): Promise { + const bytes = Math.max(0, Math.trunc(Number(input.bytes) || 0)) + const existing = this.readStorageRow() + if (!existing) { + this.ctx.storage.sql.exec( + `INSERT INTO storage_bytes_state (id, bytes, revision, updated_at) + VALUES (?, ?, 1, ?)`, + storageBytesStateRowId, + bytes, + input.updatedAt, + ) + const row = this.readStorageRow() + if (!row) { + throw new Error( + 'UserMeter setStorageBytes failed to materialize storage state.', + ) + } + return { + ...this.storageReadyState(row.bytes, row.revision), + created: true, + } + } + const nextRevision = existing.revision + 1 + this.ctx.storage.sql.exec( + `UPDATE storage_bytes_state + SET bytes = ?, + revision = ?, + updated_at = ? + WHERE id = ? AND revision = ?`, + bytes, + nextRevision, + input.updatedAt, + storageBytesStateRowId, + existing.revision, + ) + const row = this.readStorageRow() + if (!row) { + throw new Error('UserMeter setStorageBytes lost storage state.') + } + return { + ...this.storageReadyState(row.bytes, row.revision), + created: false, + } + } + async purge(): Promise<{ ok: true }> { await this.ctx.blockConcurrencyWhile(async () => { await this.ctx.storage.deleteAll() @@ -622,9 +842,22 @@ class UserMeterBase extends DurableObject { mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(revision), }) } + // Shadow is a singleton outside keyset paging — emit it once on the + // first page only so section totals and multi-page consumers do not + // double-count. + const includeStorageShadow = cursor == null + const storageRow = includeStorageShadow ? this.readStorageRow() : null const last = pageRows[pageRows.length - 1] return { counters, + storageBytesShadow: storageRow + ? { + bytes: storageRow.bytes, + revision: storageRow.revision, + updatedAt: storageRow.updatedAt, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(storageRow.revision), + } + : null, nextStartAfter: truncated && last ? encodeExportCursor({ @@ -672,6 +905,24 @@ export type UserMeterRpc = { day: string updatedAt: string }) => Promise + /** Cutover-support cold seed; expand-phase shadow uses setStorageBytes. */ + initializeStorageBytes: (input: { + bytes: number + updatedAt: string + }) => Promise + /** Cutover-support / shadow read; expand-phase usage reads D1. */ + readStorageBytes: () => Promise + /** Cutover-support reserve; expand-phase enforcement uses D1. */ + reserveStorageBytes: (input: { + requested: number + limit: number + updatedAt: string + }) => Promise + /** Expand-phase absolute shadow-set from D1. */ + setStorageBytes: (input: { + bytes: number + updatedAt: string + }) => Promise purge: () => Promise<{ ok: true }> exportCounters: (input: { pageSize?: number diff --git a/packages/worker/src/entitlements/user-meter.workers.test.ts b/packages/worker/src/entitlements/user-meter.workers.test.ts index d974b3fd63..5c1440355a 100644 --- a/packages/worker/src/entitlements/user-meter.workers.test.ts +++ b/packages/worker/src/entitlements/user-meter.workers.test.ts @@ -8,7 +8,12 @@ import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' import { EntitlementLimitError } from './errors.ts' import { planLimits } from './plans.ts' -import { consumeDailyEntitlement, refundDailyEntitlement } from './service.ts' +import { + assertWithinStorageBytesEntitlement, + consumeDailyEntitlement, + readUserD1StorageBytes, + refundDailyEntitlement, +} from './service.ts' import { ensureEntitlementTestSchema } from './test-schema.ts' import { userMeterRpc } from './user-meter-client.ts' import { UserMeter, userMeterMirrorUpdatedAtToken } from './user-meter-do.ts' @@ -555,6 +560,7 @@ test('UserMeter daily entitlement consume/refund/read/export/purge workflow is p }) expect(await meterA.exportCounters({})).toEqual({ counters: [], + storageBytesShadow: null, nextStartAfter: null, truncated: false, }) @@ -683,6 +689,7 @@ test('UserMeter purge blocks concurrent RPCs across deleteAll and schema restore expect(readDuringPurge).toEqual({ outcome: 'needs_bootstrap' }) expect(exportDuringPurge).toEqual({ counters: [], + storageBytesShadow: null, nextStartAfter: null, truncated: false, }) @@ -690,3 +697,330 @@ test('UserMeter purge blocks concurrent RPCs across deleteAll and schema restore outcome: 'needs_bootstrap', }) }, 30_000) + +test('storage bytes enforcement remains D1-authoritative even when env is passed', async () => { + const user = await seedFreeUser('meter-storage-d1-authority') + const drain = createWaitUntilDrain() + const meter = userMeterRpc({ env, userId: user.userId }) + const storageLimit = planLimits.free.maxStorageBytes + const legacyBytes = storageLimit - 10 + + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(legacyBytes, '2026-07-31T12:00:00.000Z', user.userId) + .run() + + await meter.setStorageBytes({ + bytes: storageLimit, + updatedAt: '2026-07-31T11:00:00.000Z', + }) + + await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + requested: 5, + waitUntil: drain.waitUntil, + }) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(legacyBytes + 5) + + const denied = await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + requested: 6, + waitUntil: drain.waitUntil, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(denied).toBeInstanceOf(EntitlementLimitError) + expect(denied).toMatchObject({ + details: { + resource: 'storage_bytes', + plan: 'free', + limit: storageLimit, + current: legacyBytes + 5, + }, + }) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(legacyBytes + 5) + + await drain.drain() + expect(await meter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: legacyBytes + 5, + }) +}, 30_000) + +test('UserMeter storage shadow failure cannot affect successful D1 reserves', async () => { + const user = await seedFreeUser('meter-storage-shadow-fail') + const drain = createWaitUntilDrain() + const storageLimit = planLimits.free.maxStorageBytes + + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(storageLimit - 20, '2026-07-31T12:00:00.000Z', user.userId) + .run() + + consoleWarn.mockImplementation(() => {}) + const failingMeterEnv = { + USER_METER: { + idFromName: env.USER_METER.idFromName.bind(env.USER_METER), + get() { + return { + async setStorageBytes() { + throw new Error('shadow write failed') + }, + } + }, + }, + } as unknown as typeof env + + await expect( + assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env: failingMeterEnv, + userId: user.userId, + email: user.email, + requested: 5, + waitUntil: drain.waitUntil, + }), + ).resolves.toBeUndefined() + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(storageLimit - 15) + await drain.drain() + expect(consoleWarn).toHaveBeenCalledWith( + 'entitlement-storage-bytes-shadow-failed', + expect.any(Error), + ) +}, 30_000) + +test('concurrent D1 storage reserves cannot overshoot the plan limit', async () => { + const user = await seedFreeUser('meter-storage-concurrent-d1') + const limit = planLimits.free.maxStorageBytes + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(limit - 10, '2026-07-31T15:00:00.000Z', user.userId) + .run() + + const attempts = await Promise.all( + Array.from({ length: 20 }, () => + assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + userId: user.userId, + email: user.email, + requested: 5, + }).then( + () => 'reserved' as const, + (error: unknown) => error, + ), + ), + ) + const reserved = attempts.filter((result) => result === 'reserved') + const denied = attempts.filter( + (result) => result instanceof EntitlementLimitError, + ) + expect(reserved).toHaveLength(2) + expect(denied).toHaveLength(18) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(limit) +}, 30_000) + +test('delayed UserMeter storage shadow re-reads latest D1 and cannot regress D1', async () => { + const user = await seedFreeUser('meter-storage-shadow-latest') + const drain = createWaitUntilDrain() + const meter = userMeterRpc({ env, userId: user.userId }) + + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(10, '2026-07-31T16:00:00.000Z', user.userId) + .run() + + let releaseShadow!: () => void + const shadowGate = new Promise((resolve) => { + releaseShadow = resolve + }) + let shadowReads = 0 + using _patch = withPatchedDbPrepare(env.APP_DB, (originalPrepare) => { + return ((query: string) => { + const statement = originalPrepare(query) + const isPointRead = + query.includes('SELECT d1_storage_bytes AS bytes') && + query.includes('FROM users') + const isReserve = query.includes( + 'SET d1_storage_bytes = d1_storage_bytes + ?', + ) + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: async () => { + if (isPointRead) { + shadowReads += 1 + await shadowGate + } + return await bound.first() + }, + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: bound.run.bind(bound), + } + }, + } + }) as D1Database['prepare'] + }) + + await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + requested: 5, + waitUntil: drain.waitUntil, + }) + await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + userId: user.userId, + email: user.email, + requested: 7, + }) + releaseShadow() + await drain.drain() + expect(shadowReads).toBeGreaterThanOrEqual(1) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(22) + expect(await meter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: 22, + }) +}, 30_000) + +test('UserMeter storage RPCs, export shadow field, and purge work additively', async () => { + const user = await seedFreeUser('meter-storage-export-purge') + const meter = userMeterRpc({ env, userId: user.userId }) + await meter.initializeStorageBytes({ + bytes: 42, + updatedAt: '2026-07-31T17:00:00.000Z', + }) + await meter.reserveStorageBytes({ + requested: 8, + limit: 1_000, + updatedAt: '2026-07-31T17:01:00.000Z', + }) + await meter.setStorageBytes({ + bytes: 11, + updatedAt: '2026-07-31T17:02:00.000Z', + }) + + const day = '2026-07-31' + for (const resource of [ + 'email_receives_per_day', + 'email_sends_per_day', + 'execute_calls_per_day', + 'outbound_fetches_per_day', + ] as const) { + await meter.initialize({ + resource, + day, + count: 1, + updatedAt: '2026-07-31T17:03:00.000Z', + }) + } + + const firstPage = await meter.exportCounters({ pageSize: 2 }) + expect(firstPage.counters).toHaveLength(2) + expect(firstPage.truncated).toBe(true) + expect(firstPage.nextStartAfter).toEqual(expect.any(String)) + expect(firstPage.storageBytesShadow).toEqual({ + bytes: 11, + revision: 3, + updatedAt: '2026-07-31T17:02:00.000Z', + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(3), + }) + + const secondPage = await meter.exportCounters({ + pageSize: 2, + startAfter: firstPage.nextStartAfter, + }) + expect(secondPage.counters).toHaveLength(2) + expect(secondPage.truncated).toBe(false) + expect(secondPage.nextStartAfter).toBeNull() + expect(secondPage.storageBytesShadow).toBeNull() + + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(0) + + await expect(meter.purge()).resolves.toEqual({ ok: true }) + expect(await meter.readStorageBytes()).toEqual({ + outcome: 'needs_bootstrap', + }) + expect(await meter.exportCounters({})).toEqual({ + counters: [], + storageBytesShadow: null, + nextStartAfter: null, + truncated: false, + }) +}, 30_000) + +test('missing-user storage reserve preserves EntitlementLimitError semantics on D1 path', async () => { + await ensureEntitlementTestSchema(env.APP_DB) + const missingUserId = 'a'.repeat(64) + const freeLimit = planLimits.free.maxStorageBytes + + await expect( + assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: missingUserId, + email: null, + requested: 1, + }), + ).resolves.toBeUndefined() + + const denied = await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: missingUserId, + email: null, + requested: freeLimit + 1, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(denied).toBeInstanceOf(EntitlementLimitError) + expect(denied).toMatchObject({ + details: { + resource: 'storage_bytes', + plan: 'free', + limit: freeLimit, + current: 0, + }, + }) +}, 30_000) diff --git a/packages/worker/src/mcp/memory/service.ts b/packages/worker/src/mcp/memory/service.ts index 993b24b0c9..846afba466 100644 --- a/packages/worker/src/mcp/memory/service.ts +++ b/packages/worker/src/mcp/memory/service.ts @@ -21,6 +21,7 @@ import { assertWithinStorageBytesEntitlement, estimateEntitlementStorageEntryByteDelta, } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' import { type McpMemoryRow, type MemoryRecord, @@ -72,7 +73,8 @@ type MemoryOwnerContext = { } type MemoryEnv = Pick & - Partial> + Partial> & + UserMeterEnv type MemoryUpsertInput = MemoryOwnerContext & { env: MemoryEnv @@ -86,6 +88,7 @@ type MemoryUpsertInput = MemoryOwnerContext & { dedupeKey?: string | null status?: 'active' | 'archived' verificationReference?: string | null + waitUntil?: (promise: Promise) => void } type MemoryDeleteInput = MemoryOwnerContext & { @@ -215,6 +218,7 @@ export async function upsertMemory(input: MemoryUpsertInput): Promise<{ await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -247,6 +251,7 @@ export async function upsertMemory(input: MemoryUpsertInput): Promise<{ } : null, }), + waitUntil: input.waitUntil, }) if (existing) { diff --git a/packages/worker/src/mcp/secrets/service.ts b/packages/worker/src/mcp/secrets/service.ts index e57cd35455..8440cf7ed1 100644 --- a/packages/worker/src/mcp/secrets/service.ts +++ b/packages/worker/src/mcp/secrets/service.ts @@ -39,6 +39,7 @@ import { assertWithinStorageBytesEntitlement, estimateEntitlementStorageEntryByteDelta, } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' import { type SecretMetadata, type SecretScope } from './types.ts' type SecretOwnerContext = { @@ -46,14 +47,17 @@ type SecretOwnerContext = { storageContext?: StorageContext | null } +type SecretWriteEnv = Pick & UserMeterEnv + type SaveSecretInput = SecretOwnerContext & { - env: Pick + env: SecretWriteEnv scope: SecretScope name: string value: string description?: string | null sessionExpiresAt?: string | null userEmail?: string | null + waitUntil?: (promise: Promise) => void } type ListSecretsInput = SecretOwnerContext & { @@ -126,6 +130,7 @@ export async function saveSecret( const encryptedValue = await encryptSecretValue(input.env, value) await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -152,6 +157,7 @@ export async function saveSecret( } : null, }), + waitUntil: input.waitUntil, }) const now = new Date().toISOString() await upsertSecretEntry({ @@ -189,13 +195,14 @@ export async function saveSecret( } export async function updateUserSecretForPackage(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null packageId: string name: string value: string description?: string | null + waitUntil?: (promise: Promise) => void }) { const [saved] = await updateUserSecretsForPackageAtomically({ env: input.env, @@ -209,6 +216,7 @@ export async function updateUserSecretForPackage(input: { description: input.description, }, ], + waitUntil: input.waitUntil, }) if (!saved) { throw new Error( @@ -226,7 +234,7 @@ export async function updateUserSecretForPackage(input: { * fail-closes in SQL if any secret lacks the grant. */ export async function setSecretsAtomically(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null secrets: Array<{ @@ -236,6 +244,7 @@ export async function setSecretsAtomically(input: { description?: string | null }> storageContext?: StorageContext | null + waitUntil?: (promise: Promise) => void }): Promise> { if (input.secrets.length === 0) { throw new Error('At least one secret is required.') @@ -279,6 +288,7 @@ export async function setSecretsAtomically(input: { value: secret.value, description: secret.description, })), + waitUntil: input.waitUntil, }) } @@ -293,11 +303,12 @@ export async function setSecretsAtomically(input: { description: secret.description, })), storageContext, + waitUntil: input.waitUntil, }) } export async function updateUserSecretsForPackageAtomically(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null packageId: string @@ -306,6 +317,7 @@ export async function updateUserSecretsForPackageAtomically(input: { value: string description?: string | null }> + waitUntil?: (promise: Promise) => void }): Promise> { const packageId = input.packageId.trim() if (!packageId) throw new Error('Package id is required.') @@ -371,9 +383,11 @@ export async function updateUserSecretsForPackageAtomically(input: { await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: requestedStorageBytes, + waitUntil: input.waitUntil, }) try { @@ -420,7 +434,7 @@ export async function updateUserSecretsForPackageAtomically(input: { } async function saveSecretsAtomically(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null scope: SecretScope @@ -430,6 +444,7 @@ async function saveSecretsAtomically(input: { description?: string | null }> storageContext?: StorageContext | null + waitUntil?: (promise: Promise) => void }): Promise> { const bucket = await getOrCreateSecretBucket({ db: input.env.APP_DB, @@ -502,9 +517,11 @@ async function saveSecretsAtomically(input: { } await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: requestedStorageBytes, + waitUntil: input.waitUntil, }) await upsertSecretEntriesAtomically({ diff --git a/packages/worker/src/mcp/values/service.node.test.ts b/packages/worker/src/mcp/values/service.node.test.ts index db0223efa3..fdf7dd1724 100644 --- a/packages/worker/src/mcp/values/service.node.test.ts +++ b/packages/worker/src/mcp/values/service.node.test.ts @@ -6,6 +6,13 @@ import { resolveStorageScopeOrder, } from '#mcp/storage-bindings.ts' import { deleteAllAppScopedValues } from '#worker/package-config-cleanup.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { + createInMemoryUserMeterEnv, + createWaitUntilDrain, + withPatchedDbPrepare, +} from '#worker/test-support/user-meter.ts' import { deleteValue, getValue, listValues, saveValue } from './service.ts' import { type ValueBucketRow, type ValueEntryRow } from './types.ts' @@ -663,3 +670,122 @@ test('saveValue permits underscore-prefixed ordinary value names', async () => { value: '{"provider":"widgets"}', }) }) + +test('saveValue awaits atomic D1 storage reserve; D1→UserMeter shadow is deferred and non-blocking', async () => { + const testDb = createValueTestDb() + const meter = createInMemoryUserMeterEnv() + const drain = createWaitUntilDrain() + const userId = 'a'.repeat(64) + const email = 'value-storage@example.com' + let d1StorageBytes = 100 + let failShadowPointRead = false + await meter.seedStorageBytes({ userId, bytes: 7 }) + + let releaseShadowRead!: () => void + const shadowReadGate = new Promise((resolve) => { + releaseShadowRead = resolve + }) + let shadowPointReadStarted = false + let atomicReserveCompleted = false + using _patch = withPatchedDbPrepare(testDb.db, (originalPrepare) => { + return ((query: string) => { + const normalized = query.replace(/\s+/g, ' ').trim().toLowerCase() + const statement = originalPrepare(query) + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: async () => { + if ( + normalized.includes('select plan, stripe_plan from users') && + normalized.includes('stable_user_id') + ) { + return { plan: 'pro', stripe_plan: null } as T + } + if ( + normalized.includes('select d1_storage_bytes as bytes') && + normalized.includes('from users') + ) { + if (failShadowPointRead) { + throw new Error('forced D1→UserMeter shadow read failure') + } + shadowPointReadStarted = true + await shadowReadGate + return { bytes: d1StorageBytes } as T + } + return await bound.first() + }, + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: async () => { + if ( + normalized.includes( + 'set d1_storage_bytes = d1_storage_bytes + ?', + ) + ) { + const requested = Number(params[0]) + const limit = Number(params[4]) + if (d1StorageBytes + requested <= limit) { + d1StorageBytes += requested + atomicReserveCompleted = true + return { meta: { changes: 1 } } + } + return { meta: { changes: 0 } } + } + return await bound.run() + }, + } + }, + } + }) as D1Database['prepare'] + }) + + const env = { APP_DB: testDb.db, ...meter.env } + const saved = await saveValue({ + env, + userId, + userEmail: email, + scope: 'user', + name: 'metered-value', + value: 'hello-storage', + waitUntil: drain.waitUntil, + }) + expect(saved).toMatchObject({ name: 'metered-value' }) + expect(atomicReserveCompleted).toBe(true) + expect(d1StorageBytes).toBeGreaterThan(100) + expect(shadowPointReadStarted).toBe(true) + expect(await userMeterRpc({ env, userId }).readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: 7, + }) + + releaseShadowRead() + await drain.drain() + expect(await userMeterRpc({ env, userId }).readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: d1StorageBytes, + }) + + failShadowPointRead = true + consoleWarn.mockImplementation(() => {}) + const failingDrain = createWaitUntilDrain() + const bytesBeforeFailingSave = d1StorageBytes + await expect( + saveValue({ + env, + userId, + userEmail: email, + scope: 'user', + name: 'metered-value-2', + value: 'still-ok', + waitUntil: failingDrain.waitUntil, + }), + ).resolves.toMatchObject({ name: 'metered-value-2' }) + expect(d1StorageBytes).toBeGreaterThan(bytesBeforeFailingSave) + await failingDrain.drain() + expect(consoleWarn).toHaveBeenCalledWith( + 'entitlement-storage-bytes-shadow-failed', + expect.any(Error), + ) +}) diff --git a/packages/worker/src/mcp/values/service.ts b/packages/worker/src/mcp/values/service.ts index e2becb8bfc..8dc3d81b49 100644 --- a/packages/worker/src/mcp/values/service.ts +++ b/packages/worker/src/mcp/values/service.ts @@ -21,6 +21,7 @@ import { assertWithinStorageBytesEntitlement, estimateEntitlementStorageEntryByteDelta, } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' type ValueOwnerContext = { userId: string @@ -28,13 +29,14 @@ type ValueOwnerContext = { } type SaveValueInput = ValueOwnerContext & { - env: Pick + env: Pick & UserMeterEnv scope: ValueScope name: string value: string description?: string | null sessionExpiresAt?: string | null userEmail?: string | null + waitUntil?: (promise: Promise) => void } type ListValuesInput = ValueOwnerContext & { @@ -87,6 +89,7 @@ export async function saveValue(input: SaveValueInput): Promise { }) await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -104,6 +107,7 @@ export async function saveValue(input: SaveValueInput): Promise { } : null, }), + waitUntil: input.waitUntil, }) const now = new Date().toISOString() await upsertValueEntry({ diff --git a/packages/worker/src/package-registry/service.node.test.ts b/packages/worker/src/package-registry/service.node.test.ts index 14b86f6dc6..83ba2c744e 100644 --- a/packages/worker/src/package-registry/service.node.test.ts +++ b/packages/worker/src/package-registry/service.node.test.ts @@ -1095,6 +1095,9 @@ function createEntitlementsDatabase(input: { const user = users.find((row) => row.email === params[0]) return (user ? { plan: user.plan } : null) as T | null } + if (query.includes('SELECT d1_storage_bytes AS bytes')) { + return null + } if ( query.includes('SELECT COUNT(*) AS count FROM saved_packages') ) { diff --git a/packages/worker/src/package-registry/service.ts b/packages/worker/src/package-registry/service.ts index c0e4790e48..673946208f 100644 --- a/packages/worker/src/package-registry/service.ts +++ b/packages/worker/src/package-registry/service.ts @@ -217,6 +217,7 @@ export async function refreshSavedPackageProjection(input: { packageId: string sourceId: string rebuildArtifacts?: boolean + waitUntil?: (promise: Promise) => void }) { return await withAccountWriteLease({ db: input.env.APP_DB, @@ -248,6 +249,7 @@ export async function refreshSavedPackageProjection(input: { }) await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -276,6 +278,7 @@ export async function refreshSavedPackageProjection(input: { } : null, }), + waitUntil: input.waitUntil, }) if (existing) { await updateSavedPackage(input.env.APP_DB, { diff --git a/packages/worker/src/storage-runner.entitlement.node.test.ts b/packages/worker/src/storage-runner.entitlement.node.test.ts index 5b591fea55..d7c51bb55a 100644 --- a/packages/worker/src/storage-runner.entitlement.node.test.ts +++ b/packages/worker/src/storage-runner.entitlement.node.test.ts @@ -229,3 +229,52 @@ test('an unreachable peer bucket with a stored estimate never blocks a write', a // Only the write target was measured live; no peer fan-out happened. expect(probedStorageIds).toEqual(['package:target']) }) + +test('StorageRunner baseline composes D1 payload with bucket estimates via check-only getCurrent', async () => { + const { planLimits } = await import('#worker/entitlements/plans.ts') + const { isEntitlementLimitError } = + await import('#worker/entitlements/errors.ts') + const userId = 'user-1' + const limit = planLimits.free.maxStorageBytes + mockModule.readUserD1StorageBytes.mockImplementation( + async (input: { userId: string }) => { + expect(input.userId).toBe(userId) + return limit - 100 + }, + ) + mockModule.listUserStorageBucketEstimates.mockResolvedValue([ + { storageId: 'package:peer', estimatedBytes: 50 }, + { storageId: 'package:target', estimatedBytes: 200 }, + ]) + const probedStorageIds: Array = [] + const getEstimatedBytes = async (storageId: string) => { + probedStorageIds.push(storageId) + return { estimatedBytes: storageId === 'package:target' ? 60 : 50 } + } + + const denied = await assertStorageRunnerWriteWithinEntitlement({ + env: createEstimateEnv(getEstimatedBytes), + userId, + email: null, + storageId: 'package:target', + requested: 1, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(isEntitlementLimitError(denied)).toBe(true) + expect(denied).toMatchObject({ + details: { + resource: 'storage_bytes', + current: limit - 100 + 50 + 60, + limit, + }, + }) + expect(probedStorageIds).toEqual(['package:target']) + expect(mockModule.readUserD1StorageBytes).toHaveBeenCalledWith( + expect.objectContaining({ + userId, + db: expect.anything(), + }), + ) +}) diff --git a/packages/worker/src/storage-runner.workers.test.ts b/packages/worker/src/storage-runner.workers.test.ts index 5f9859b213..c606da2f18 100644 --- a/packages/worker/src/storage-runner.workers.test.ts +++ b/packages/worker/src/storage-runner.workers.test.ts @@ -285,6 +285,7 @@ test('storage runner storage byte entitlement aggregates only inventoried user b .bind(targetD1Bytes, new Date().toISOString(), userId) .run() expect(initialD1Bytes).toBe(0) + // D1 remains the sole payload authority for composed getCurrent. await expect( readUserD1StorageBytes({ db: env.APP_DB, userId }), ).resolves.toBe(targetD1Bytes) diff --git a/packages/worker/src/test-support/user-meter.ts b/packages/worker/src/test-support/user-meter.ts index bf3a390f23..8452d2d284 100644 --- a/packages/worker/src/test-support/user-meter.ts +++ b/packages/worker/src/test-support/user-meter.ts @@ -2,10 +2,12 @@ import { isDailyEntitlementResource, userMeterMirrorUpdatedAtToken, type DailyEntitlementResource, + type UserMeterStorageBytesState, } from '#worker/entitlements/user-meter-do.ts' import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' type MeterRow = { count: number; revision: number } +type StorageRow = { bytes: number; revision: number; updatedAt: string } /** * In-memory UserMeter stub keyed by `idFromName` (stable userId) for node-unit @@ -14,6 +16,7 @@ type MeterRow = { count: number; revision: number } */ export function createInMemoryUserMeterEnv() { const metersByUser = new Map>() + const storageByUser = new Map() function counterKey(resource: string, day: string) { return `${resource}\0${day}` @@ -37,6 +40,15 @@ export function createInMemoryUserMeterEnv() { } } + function storageReady(row: StorageRow) { + return { + outcome: 'ready' as const, + bytes: row.bytes, + revision: row.revision, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(row.revision), + } + } + return { async initialize(input: { resource: string @@ -105,11 +117,77 @@ export function createInMemoryUserMeterEnv() { rows.set(counterKey(input.resource, input.day), next) return ready(next) }, + async initializeStorageBytes(input: { + bytes: number + updatedAt: string + }) { + const existing = storageByUser.get(userId) + if (existing) return { ...storageReady(existing), created: false } + const row = { + bytes: Math.max(0, Math.trunc(input.bytes) || 0), + revision: 1, + updatedAt: input.updatedAt, + } + storageByUser.set(userId, row) + return { ...storageReady(row), created: true } + }, + async readStorageBytes() { + const existing = storageByUser.get(userId) + if (!existing) return { outcome: 'needs_bootstrap' as const } + return storageReady(existing) + }, + async reserveStorageBytes(input: { + requested: number + limit: number + updatedAt: string + }) { + const requested = Math.max(0, Math.trunc(input.requested) || 0) + const existing = storageByUser.get(userId) + if (!existing) return { outcome: 'needs_bootstrap' as const } + if ( + requested > 0 && + (input.limit < 1 || existing.bytes + requested > input.limit) + ) { + return { ...storageReady(existing), reserved: false } + } + if (requested === 0) { + return { ...storageReady(existing), reserved: true } + } + const next = { + bytes: existing.bytes + requested, + revision: existing.revision + 1, + updatedAt: input.updatedAt, + } + storageByUser.set(userId, next) + return { ...storageReady(next), reserved: true } + }, + async setStorageBytes(input: { bytes: number; updatedAt: string }) { + const bytes = Math.max(0, Math.trunc(input.bytes) || 0) + const existing = storageByUser.get(userId) + if (!existing) { + const row = { bytes, revision: 1, updatedAt: input.updatedAt } + storageByUser.set(userId, row) + return { ...storageReady(row), created: true } + } + const next = { + bytes, + revision: existing.revision + 1, + updatedAt: input.updatedAt, + } + storageByUser.set(userId, next) + return { ...storageReady(next), created: false } + }, async purge() { rows.clear() + storageByUser.delete(userId) return { ok: true as const } }, - async exportCounters() { + async exportCounters( + input: { + pageSize?: number + startAfter?: string | null + } = {}, + ) { const counters = [...rows.entries()].map(([entryKey, row]) => { const [resource, day] = entryKey.split('\0') return { @@ -121,7 +199,25 @@ export function createInMemoryUserMeterEnv() { mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(row.revision), } }) - return { counters, nextStartAfter: null, truncated: false } + const includeStorageShadow = + typeof input.startAfter !== 'string' || input.startAfter.length === 0 + const storage = includeStorageShadow + ? storageByUser.get(userId) + : undefined + const storageBytesShadow: UserMeterStorageBytesState | null = storage + ? { + bytes: storage.bytes, + revision: storage.revision, + updatedAt: storage.updatedAt, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(storage.revision), + } + : null + return { + counters, + storageBytesShadow, + nextStartAfter: null, + truncated: false, + } }, } } @@ -136,6 +232,7 @@ export function createInMemoryUserMeterEnv() { return { env, metersByUser, + storageByUser, async seed(input: { userId: string resource: DailyEntitlementResource @@ -149,6 +246,16 @@ export function createInMemoryUserMeterEnv() { updatedAt: new Date().toISOString(), }) }, + async seedStorageBytes(input: { + userId: string + bytes: number + updatedAt?: string + }) { + await meterFor(input.userId).initializeStorageBytes({ + bytes: input.bytes, + updatedAt: input.updatedAt ?? new Date().toISOString(), + }) + }, } }