From 3c2114890278003b2b5eb354e8e7ab69d836736f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 31 Jul 2026 22:46:43 +0000 Subject: [PATCH 01/15] perf(entitlements): cache enforcement plan lookups Co-authored-by: Kent C. Dodds --- .../contributing/architecture/entitlements.md | 15 +++++- .../entitlements/entitlements.node.test.ts | 52 +++++++++++++++++++ packages/worker/src/entitlements/service.ts | 9 ++-- 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index 916671ffbc..c0507cb62d 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -16,8 +16,9 @@ Module: `packages/worker/src/entitlements/` `resolveEffectivePlan(manual, stripe)`. - `errors.ts` — the one typed error (`EntitlementLimitError`) and the one user-facing message builder every enforcement point uses. -- `service.ts` — `getUserPlan`, `assertWithinEntitlement`, built-in D1 usage - counters, and the daily-counter helpers for rate-style limits. +- `service.ts` — `getUserPlan`, `getCachedUserPlan` (60s TTL enforcement cache), + `assertWithinEntitlement`, built-in D1 usage counters, and the daily-counter + helpers for rate-style limits. ## Plan model @@ -190,6 +191,16 @@ use stable-id reverse resolution. `findUserAccountByStableUserId` in verified-state (for example the outbound fetch gateway), mirroring `findUserAccount` in `email/platform-address.ts`. +**Enforcement plan cache:** `assertWithinEntitlement` resolves the plan limit +via `getCachedUserPlan`, which wraps `getUserPlan` behind a 60-second per-isolate +TTL cache keyed by D1 binding and `(stable_user_id, normalized email)`. Cache +entries share the lookup semantics above; failures are never cached. Built-in +usage counters and any `getCurrent` override are read fresh on every call, so +only the plan limit can be stale — a plan change may take up to ~60s to bind at +enforcement points while current usage stays accurate. Surfaces that display the +user's plan (billing UI, email usage) should keep calling `getUserPlan` +directly. + ## The error shape Every enforcement point throws `EntitlementLimitError` from diff --git a/packages/worker/src/entitlements/entitlements.node.test.ts b/packages/worker/src/entitlements/entitlements.node.test.ts index 3b89d4cb87..2b3585ce6b 100644 --- a/packages/worker/src/entitlements/entitlements.node.test.ts +++ b/packages/worker/src/entitlements/entitlements.node.test.ts @@ -596,6 +596,58 @@ test('assertWithinEntitlement passes under the limit, throws at it, and enforces expect(error.message).toBe(buildEntitlementLimitMessage(error.details)) }) +test('assertWithinEntitlement reuses cached plan within TTL while still enforcing usage', async () => { + const userId = await createStableUserIdFromEmail(plannedEmail) + const freeLimit = planLimits.free.maxScheduledJobs + const counts = { jobs: freeLimit - 1 } + const { db, queries } = createEntitlementsTestDb({ + users: [{ email: plannedEmail, plan: 'free', stable_user_id: userId }], + counts, + }) + const planQueries = () => + queries.filter((query) => + query.sql.includes('email = ? AND stable_user_id = ?'), + ) + const usageQueries = () => + queries.filter((query) => query.sql.includes('FROM jobs')) + + await assertWithinEntitlement({ + db, + userId, + email: plannedEmail, + resource: 'scheduled_jobs', + }) + await assertWithinEntitlement({ + db, + userId, + email: plannedEmail, + resource: 'scheduled_jobs', + }) + expect(planQueries()).toHaveLength(1) + expect(usageQueries()).toHaveLength(2) + + counts.jobs = freeLimit + const denied = await assertWithinEntitlement({ + db, + userId, + email: plannedEmail, + resource: 'scheduled_jobs', + }).then( + () => null, + (thrown: unknown) => thrown, + ) + if (!(denied instanceof EntitlementLimitError)) { + throw new Error('Expected an EntitlementLimitError.') + } + expect(denied.details).toMatchObject({ + plan: 'free', + limit: freeLimit, + current: freeLimit, + }) + expect(planQueries()).toHaveLength(1) + expect(usageQueries()).toHaveLength(3) +}) + test('assertWithinEntitlement enforces free concurrent workflow limit without email', async () => { const freeLimit = planLimits.free.maxConcurrentWorkflows const { db } = createEntitlementsTestDb({ diff --git a/packages/worker/src/entitlements/service.ts b/packages/worker/src/entitlements/service.ts index f528cbd6c4..c7a7f534a1 100644 --- a/packages/worker/src/entitlements/service.ts +++ b/packages/worker/src/entitlements/service.ts @@ -791,8 +791,9 @@ export type AssertWithinEntitlementInput = { /** * Account email of the acting user when available. When present, plan * lookup requires the email + stable-id pair. When absent (package-job / - * workflow contexts with `email: ''`), {@link getUserPlan} reverse-resolves - * by stable userId; unknown accounts still fail closed to `free`. + * workflow contexts with `email: ''`), {@link getCachedUserPlan} + * reverse-resolves by stable userId; unknown accounts still fail closed + * to `free`. */ email: string | null | undefined resource: EntitlementResource @@ -812,7 +813,9 @@ export type AssertWithinEntitlementInput = { export async function assertWithinEntitlement( input: AssertWithinEntitlementInput, ): Promise { - const plan = await getUserPlan(input.db, { + // Cached plan resolution: quota limit lookup tolerates short-TTL + // staleness; usage counters below stay fresh on every call. + const plan = await getCachedUserPlan(input.db, { userId: input.userId, email: input.email, }) From 7d5d77741135a5d9d071cdc68d51b9a4579a3a1f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 31 Jul 2026 22:54:38 +0000 Subject: [PATCH 02/15] style(docs): format entitlement cache notes Co-authored-by: Kent C. Dodds --- docs/contributing/architecture/entitlements.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index c0507cb62d..5b66773a99 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -192,14 +192,14 @@ verified-state (for example the outbound fetch gateway), mirroring `findUserAccount` in `email/platform-address.ts`. **Enforcement plan cache:** `assertWithinEntitlement` resolves the plan limit -via `getCachedUserPlan`, which wraps `getUserPlan` behind a 60-second per-isolate -TTL cache keyed by D1 binding and `(stable_user_id, normalized email)`. Cache -entries share the lookup semantics above; failures are never cached. Built-in -usage counters and any `getCurrent` override are read fresh on every call, so -only the plan limit can be stale — a plan change may take up to ~60s to bind at -enforcement points while current usage stays accurate. Surfaces that display the -user's plan (billing UI, email usage) should keep calling `getUserPlan` -directly. +via `getCachedUserPlan`, which wraps `getUserPlan` behind a 60-second +per-isolate TTL cache keyed by D1 binding and +`(stable_user_id, normalized email)`. Cache entries share the lookup semantics +above; failures are never cached. Built-in usage counters and any `getCurrent` +override are read fresh on every call, so only the plan limit can be stale — a +plan change may take up to ~60s to bind at enforcement points while current +usage stays accurate. Surfaces that display the user's plan (billing UI, email +usage) should keep calling `getUserPlan` directly. ## The error shape From 59a82385100195c41ce4a57ab35ae4cec5531e3a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 00:11:34 +0000 Subject: [PATCH 03/15] feat(entitlements): add per-user quota meter Co-authored-by: Kent C. Dodds --- .../src/admin/user-usage-data.node.test.ts | 187 ++++- packages/worker/src/admin/user-usage-data.ts | 11 +- .../src/app/account-usage-data.node.test.ts | 252 +++++-- packages/worker/src/app/account-usage-data.ts | 11 +- packages/worker/src/email/inbound-delivery.ts | 181 +++-- .../inbound-entitlements.workers.test.ts | 588 ++++++++++++++- packages/worker/src/email/inbound.ts | 8 + .../worker/src/email/inbound.workers.test.ts | 68 +- packages/worker/src/email/outbound.ts | 7 +- .../worker/src/email/outbound.workers.test.ts | 92 ++- .../entitlements/entitlements.node.test.ts | 111 ++- packages/worker/src/entitlements/service.ts | 358 +++++++-- .../src/entitlements/user-meter-client.ts | 28 + .../worker/src/entitlements/user-meter-do.ts | 678 ++++++++++++++++++ .../entitlements/user-meter.workers.test.ts | 616 ++++++++++++++++ packages/worker/src/index.ts | 2 + .../mcp/capabilities/email/email-usage-get.ts | 7 +- .../email/email-usage-get.workers.test.ts | 235 +++--- .../worker/src/mcp/fetch-gateway.node.test.ts | 18 +- packages/worker/src/mcp/fetch-gateway.ts | 4 +- .../src/mcp/fetch-gateway.workers.test.ts | 63 +- packages/worker/src/mcp/tools/execute.ts | 2 + .../worker/src/test-support/user-meter.ts | 182 +++++ ...er-scoped-durable-object-name.node.test.ts | 9 + .../src/user-scoped-durable-object-name.ts | 5 + packages/worker/worker-configuration.d.ts | 9 +- packages/worker/wrangler.jsonc | 16 + 27 files changed, 3293 insertions(+), 455 deletions(-) create mode 100644 packages/worker/src/entitlements/user-meter-client.ts create mode 100644 packages/worker/src/entitlements/user-meter-do.ts create mode 100644 packages/worker/src/entitlements/user-meter.workers.test.ts create mode 100644 packages/worker/src/test-support/user-meter.ts diff --git a/packages/worker/src/admin/user-usage-data.node.test.ts b/packages/worker/src/admin/user-usage-data.node.test.ts index b7e91e8228..cb538919ec 100644 --- a/packages/worker/src/admin/user-usage-data.node.test.ts +++ b/packages/worker/src/admin/user-usage-data.node.test.ts @@ -1,8 +1,15 @@ import { expect, test } from 'vitest' +import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { type AdminUsageRollup } from '#app/loader-data.ts' import { loadAdminUserUsageData } from './user-usage-data.ts' +function withUserMeter(env: { APP_DB: D1Database } & Record) { + const meter = createInMemoryUserMeterEnv() + return { ...env, ...meter.env, meter } +} + type UserRow = { id: number username: string @@ -161,7 +168,7 @@ test('loadAdminUserUsageData returns null for unknown users and zeroed usage for const emptyDb = createAdminUserUsageTestDb({ users: [] }) expect( await loadAdminUserUsageData( - { APP_DB: emptyDb } as Env, + withUserMeter({ APP_DB: emptyDb }) as Env, 'missing-stable-user', new Date('2026-07-05T12:00:00.000Z'), ), @@ -185,7 +192,7 @@ test('loadAdminUserUsageData returns null for unknown users and zeroed usage for }) const data = await loadAdminUserUsageData( - { APP_DB: db } as Env, + withUserMeter({ APP_DB: db }) as Env, usageUserId, new Date('2026-07-05T12:00:00.000Z'), ) @@ -254,7 +261,7 @@ test('loadAdminUserUsageData warns above eighty percent of plan limits', async ( }) const data = await loadAdminUserUsageData( - { APP_DB: db } as Env, + withUserMeter({ APP_DB: db }) as Env, usageUserId, new Date('2026-07-05T12:00:00.000Z'), ) @@ -299,7 +306,7 @@ test('loadAdminUserUsageData rejects an invalid stored plan', async () => { await expect( loadAdminUserUsageData( - { APP_DB: db } as Env, + withUserMeter({ APP_DB: db }) as Env, usageUserId, new Date('2026-07-05T12:00:00.000Z'), ), @@ -361,7 +368,10 @@ test('loadAdminUserUsageData caches rollup reads in KV and serves repeat loads f }, }) const { kv, store } = createFakeKv() - const env = { APP_DB: countingDb, BUNDLE_ARTIFACTS_KV: kv } as Env + const env = withUserMeter({ + APP_DB: countingDb, + BUNDLE_ARTIFACTS_KV: kv, + }) as Env const now = new Date('2026-07-05T12:00:00.000Z') const first = await loadAdminUserUsageData(env, usageUserId, now) @@ -413,7 +423,7 @@ test('loadAdminUserUsageData keeps current-month and month-over-month rollups on }) const data = await loadAdminUserUsageData( - { APP_DB: db } as Env, + withUserMeter({ APP_DB: db }) as Env, usageUserId, new Date('2026-07-01T00:00:00.000Z'), ) @@ -427,6 +437,171 @@ test('loadAdminUserUsageData keeps current-month and month-over-month rollups on expect(getEventCount(data?.monthUsage[1]?.usage, 'execute')).toBe(12) }) +test('loadAdminUserUsageData reads daily counts from UserMeter (bootstrap then warm)', async () => { + const now = new Date('2026-07-05T12:00:00.000Z') + const day = utcDayKey(now) + + const bootstrapEmail = 'bootstrap-drilldown@example.com' + const bootstrapUserId = await createStableUserIdFromEmail(bootstrapEmail) + const bootstrapped = await loadAdminUserUsageData( + withUserMeter({ + APP_DB: createAdminUserUsageTestDb({ + users: [ + { + id: 3, + username: 'bootstrap', + email: bootstrapEmail, + plan: 'pro', + stable_user_id: bootstrapUserId, + }, + ], + dailyCounters: [ + { + user_id: bootstrapUserId, + resource: 'email_receives_per_day', + day, + count: 55, + }, + { + user_id: bootstrapUserId, + resource: 'outbound_fetches_per_day', + day, + count: 66, + }, + ], + resourceCounts: { + [bootstrapUserId]: { secrets: 3 }, + }, + }), + }) as Env, + bootstrapUserId, + now, + ) + expect( + bootstrapped?.entitlementConsumption.find( + (row) => row.resource === 'email_receives_per_day', + )?.current, + ).toBe(55) + expect( + bootstrapped?.entitlementConsumption.find( + (row) => row.resource === 'outbound_fetches_per_day', + )?.current, + ).toBe(66) + expect( + bootstrapped?.entitlementConsumption.find( + (row) => row.resource === 'secrets', + )?.current, + ).toBe(3) + + const meterEmail = 'meter-drilldown@example.com' + const meterUserId = await createStableUserIdFromEmail(meterEmail) + const meter = createInMemoryUserMeterEnv() + await meter.seed({ + userId: meterUserId, + resource: 'email_sends_per_day', + day, + count: 111, + }) + await meter.seed({ + userId: meterUserId, + resource: 'email_receives_per_day', + day, + count: 222, + }) + await meter.seed({ + userId: meterUserId, + resource: 'execute_calls_per_day', + day, + count: 333, + }) + await meter.seed({ + userId: meterUserId, + resource: 'outbound_fetches_per_day', + day, + count: 444, + }) + const warm = await loadAdminUserUsageData( + { + APP_DB: createAdminUserUsageTestDb({ + users: [ + { + id: 4, + username: 'meter', + email: meterEmail, + plan: 'pro', + stable_user_id: meterUserId, + }, + ], + dailyCounters: [ + { + user_id: meterUserId, + resource: 'email_sends_per_day', + day, + count: 11, + }, + { + user_id: meterUserId, + resource: 'email_receives_per_day', + day, + count: 22, + }, + { + user_id: meterUserId, + resource: 'execute_calls_per_day', + day, + count: 33, + }, + { + user_id: meterUserId, + resource: 'outbound_fetches_per_day', + day, + count: 44, + }, + ], + resourceCounts: { + [meterUserId]: { + saved_packages: 6, + stored_email_messages: 9, + }, + }, + }), + ...meter.env, + } as Env, + meterUserId, + now, + ) + expect( + warm?.entitlementConsumption.find( + (row) => row.resource === 'email_sends_per_day', + )?.current, + ).toBe(111) + expect( + warm?.entitlementConsumption.find( + (row) => row.resource === 'email_receives_per_day', + )?.current, + ).toBe(222) + expect( + warm?.entitlementConsumption.find( + (row) => row.resource === 'execute_calls_per_day', + )?.current, + ).toBe(333) + expect( + warm?.entitlementConsumption.find( + (row) => row.resource === 'outbound_fetches_per_day', + )?.current, + ).toBe(444) + expect( + warm?.entitlementConsumption.find( + (row) => row.resource === 'saved_packages', + )?.current, + ).toBe(6) + expect( + warm?.entitlementConsumption.find( + (row) => row.resource === 'stored_email_messages', + )?.current, + ).toBe(9) +}) + function getEventCount( usage: Array | undefined, metric: AdminUsageRollup['metric'], diff --git a/packages/worker/src/admin/user-usage-data.ts b/packages/worker/src/admin/user-usage-data.ts index 9a1c29deea..221fdbd277 100644 --- a/packages/worker/src/admin/user-usage-data.ts +++ b/packages/worker/src/admin/user-usage-data.ts @@ -8,7 +8,7 @@ import { type EntitlementResource, type PlanName, } from '#worker/entitlements/plans.ts' -import { readEntitlementResourceUsage } from '#worker/entitlements/service.ts' +import { readCurrentEntitlementResourceUsage } from '#worker/entitlements/service.ts' import { createKvCachifiedCache } from '#worker/kv-cachified.ts' import { resolveUserStableId } from '#worker/user-id.ts' import { @@ -113,7 +113,7 @@ export async function loadAdminUserUsageData( currentMonth, }), readEntitlementConsumption({ - db: env.APP_DB, + env, usageUserId, plan, now, @@ -140,15 +140,16 @@ export async function loadAdminUserUsageData( } async function readEntitlementConsumption(input: { - db: D1Database + env: Env usageUserId: string plan: PlanName now: Date }): Promise> { return await Promise.all( adminUserUsageEntitlementResources.map(async (resource) => { - const current = await readEntitlementResourceUsage({ - db: input.db, + const current = await readCurrentEntitlementResourceUsage({ + db: input.env.APP_DB, + env: input.env, userId: input.usageUserId, resource, now: input.now, diff --git a/packages/worker/src/app/account-usage-data.node.test.ts b/packages/worker/src/app/account-usage-data.node.test.ts index 0281b04c4e..fe49db6938 100644 --- a/packages/worker/src/app/account-usage-data.node.test.ts +++ b/packages/worker/src/app/account-usage-data.node.test.ts @@ -1,6 +1,15 @@ import { expect, test } from 'vitest' -import { loadAccountUsageData } from '#app/account-usage-data.ts' +import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' +import { loadAccountUsageData } from '#app/account-usage-data.ts' + +type DailyCounterRow = { + user_id: string + resource: string + day: string + count: number +} function createUsageTestDb(input: { userId: number @@ -8,73 +17,192 @@ function createUsageTestDb(input: { plan: string stripePlan?: string | null packageCount?: number + dailyCounters?: Array }) { const stableUserId = testStableUserIdFromEmail(input.email) + const dailyCounters = input.dailyCounters?.map((row) => ({ ...row })) ?? [] return { - prepare(query: string) { - const normalized = query.replace(/\s+/g, ' ').trim().toLowerCase() - return { - bind(...params: Array) { - return { - async first() { - if ( - normalized.includes('from users') && - normalized.includes('where id') - ) { - return { - id: input.userId, - plan: input.plan, - stripe_plan: input.stripePlan ?? null, - stable_user_id: stableUserId, - } as T - } - if (normalized.includes('from saved_packages')) { - return { count: input.packageCount ?? 0 } as T - } - if (normalized.includes('from entitlement_daily_counters')) { - return { count: 0 } as T - } - if ( - normalized.includes('count(*)') || - normalized.includes('sum(') - ) { - return { count: 0, total: 0 } as T - } - void params - return null - }, - async all() { - return { results: [] } - }, - } - }, - } - }, - } as unknown as D1Database + stableUserId, + db: { + prepare(query: string) { + const normalized = query.replace(/\s+/g, ' ').trim().toLowerCase() + return { + bind(...params: Array) { + return { + async first() { + if ( + normalized.includes('from users') && + normalized.includes('where id') + ) { + return { + id: input.userId, + plan: input.plan, + stripe_plan: input.stripePlan ?? null, + stable_user_id: stableUserId, + } as T + } + if (normalized.includes('from saved_packages')) { + return { count: input.packageCount ?? 0 } as T + } + if (normalized.includes('from entitlement_daily_counters')) { + const row = dailyCounters.find( + (counter) => + counter.user_id === params[0] && + counter.resource === params[1] && + counter.day === params[2], + ) + return (row ? { count: row.count } : { count: 0 }) as T + } + if ( + normalized.includes('count(*)') || + normalized.includes('sum(') || + normalized.includes('d1_storage_bytes') + ) { + return { count: 0, total: 0, bytes: 0 } as T + } + void params + return null + }, + async all() { + return { results: [] } + }, + } + }, + } + }, + } as unknown as D1Database, + } } -test('loadAccountUsageData returns plan and entitlement rows for the account', async () => { - const env = { - APP_DB: createUsageTestDb({ - userId: 7, - email: 'usage@example.com', - plan: 'free', - packageCount: 2, - }), - } as Env +function currentFor( + data: Awaited>, + resource: string, +) { + return data?.entitlementConsumption.find((row) => row.resource === resource) + ?.current +} - const data = await loadAccountUsageData({ - env, +test('loadAccountUsageData returns plan rows and authoritative UserMeter daily counts', async () => { + const now = new Date('2026-07-25T12:00:00.000Z') + const day = utcDayKey(now) + + const baselineMeter = createInMemoryUserMeterEnv() + const { db: emptyDailyDb } = createUsageTestDb({ + userId: 7, + email: 'usage@example.com', + plan: 'free', + packageCount: 2, + }) + const baseline = await loadAccountUsageData({ + env: { APP_DB: emptyDailyDb, ...baselineMeter.env } as Env, userId: 7, - now: new Date('2026-07-25T12:00:00.000Z'), + now, + }) + expect(baseline?.ok).toBe(true) + expect(baseline?.plan).toBe('free') + expect(baseline?.today).toBe('2026-07-25') + expect(currentFor(baseline, 'saved_packages')).toBe(2) + expect(baseline?.entitlementConsumption.length).toBeGreaterThan(5) + + const bootstrapEmail = 'usage-bootstrap@example.com' + const bootstrapUserId = testStableUserIdFromEmail(bootstrapEmail) + const bootstrapMeter = createInMemoryUserMeterEnv() + const { db: bootstrapDb } = createUsageTestDb({ + userId: 8, + email: bootstrapEmail, + plan: 'pro', + packageCount: 1, + dailyCounters: [ + { + user_id: bootstrapUserId, + resource: 'email_sends_per_day', + day, + count: 17, + }, + { + user_id: bootstrapUserId, + resource: 'execute_calls_per_day', + day, + count: 91, + }, + ], + }) + const bootstrapped = await loadAccountUsageData({ + env: { APP_DB: bootstrapDb, ...bootstrapMeter.env } as Env, + userId: 8, + now, + }) + expect(currentFor(bootstrapped, 'email_sends_per_day')).toBe(17) + expect(currentFor(bootstrapped, 'execute_calls_per_day')).toBe(91) + expect(currentFor(bootstrapped, 'saved_packages')).toBe(1) + + const meterEmail = 'usage-meter@example.com' + const meterUserId = testStableUserIdFromEmail(meterEmail) + const warmMeter = createInMemoryUserMeterEnv() + const { db: warmDb } = createUsageTestDb({ + userId: 9, + email: meterEmail, + plan: 'pro', + packageCount: 4, + dailyCounters: [ + { + user_id: meterUserId, + resource: 'email_sends_per_day', + day, + count: 11, + }, + { + user_id: meterUserId, + resource: 'email_receives_per_day', + day, + count: 22, + }, + { + user_id: meterUserId, + resource: 'execute_calls_per_day', + day, + count: 33, + }, + { + user_id: meterUserId, + resource: 'outbound_fetches_per_day', + day, + count: 44, + }, + ], + }) + await warmMeter.seed({ + userId: meterUserId, + resource: 'email_sends_per_day', + day, + count: 101, + }) + await warmMeter.seed({ + userId: meterUserId, + resource: 'email_receives_per_day', + day, + count: 202, + }) + await warmMeter.seed({ + userId: meterUserId, + resource: 'execute_calls_per_day', + day, + count: 303, + }) + await warmMeter.seed({ + userId: meterUserId, + resource: 'outbound_fetches_per_day', + day, + count: 404, + }) + const authoritative = await loadAccountUsageData({ + env: { APP_DB: warmDb, ...warmMeter.env } as Env, + userId: 9, + now, }) - expect(data?.ok).toBe(true) - expect(data?.plan).toBe('free') - expect(data?.today).toBe('2026-07-25') - const packages = data?.entitlementConsumption.find( - (row) => row.resource === 'saved_packages', - ) - expect(packages?.current).toBe(2) - expect(packages?.limit).toBeGreaterThan(0) - expect(data?.entitlementConsumption.length).toBeGreaterThan(5) + expect(currentFor(authoritative, 'email_sends_per_day')).toBe(101) + expect(currentFor(authoritative, 'email_receives_per_day')).toBe(202) + expect(currentFor(authoritative, 'execute_calls_per_day')).toBe(303) + expect(currentFor(authoritative, 'outbound_fetches_per_day')).toBe(404) + expect(currentFor(authoritative, 'saved_packages')).toBe(4) }) diff --git a/packages/worker/src/app/account-usage-data.ts b/packages/worker/src/app/account-usage-data.ts index 6a41dd74d6..fe416ee63d 100644 --- a/packages/worker/src/app/account-usage-data.ts +++ b/packages/worker/src/app/account-usage-data.ts @@ -7,7 +7,7 @@ import { type EntitlementResource, type PlanName, } from '#worker/entitlements/plans.ts' -import { readEntitlementResourceUsage } from '#worker/entitlements/service.ts' +import { readCurrentEntitlementResourceUsage } from '#worker/entitlements/service.ts' import { resolveUserStableId } from '#worker/user-id.ts' import { type AccountUsageEntitlementConsumption, @@ -62,7 +62,7 @@ export async function loadAccountUsageData(input: { ) const usageUserId = resolveUserStableId(row) const entitlementConsumption = await readEntitlementConsumption({ - db: input.env.APP_DB, + env: input.env, usageUserId, plan, now, @@ -78,15 +78,16 @@ export async function loadAccountUsageData(input: { } async function readEntitlementConsumption(input: { - db: D1Database + env: Env usageUserId: string plan: PlanName now: Date }): Promise> { return await Promise.all( accountUsageEntitlementResources.map(async (resource) => { - const current = await readEntitlementResourceUsage({ - db: input.db, + const current = await readCurrentEntitlementResourceUsage({ + db: input.env.APP_DB, + env: input.env, userId: input.usageUserId, resource, now: input.now, diff --git a/packages/worker/src/email/inbound-delivery.ts b/packages/worker/src/email/inbound-delivery.ts index 6102a1df1d..f5499bf1fb 100644 --- a/packages/worker/src/email/inbound-delivery.ts +++ b/packages/worker/src/email/inbound-delivery.ts @@ -5,6 +5,11 @@ import { EntitlementLimitError, } from '#worker/entitlements/errors.ts' import { type PlanName } from '#worker/entitlements/plans.ts' +import { scheduleAbsoluteDailyEntitlementMirror } from '#worker/entitlements/service.ts' +import { + userMeterRpc, + type UserMeterEnv, +} from '#worker/entitlements/user-meter-client.ts' import { normalizeEmailAddress } from './address.ts' import { emailRawMimeKey, @@ -447,17 +452,47 @@ async function readCounter(input: { return Number(row?.count ?? 0) } +/** Point-read today's user inbound receive count from UserMeter. */ export async function readUserInboundReceiveCount(input: { db: D1Database + env: UserMeterEnv userId: string day: string + now?: Date }) { - return await readCounter({ - db: input.db, - table: 'entitlement_daily_counters', - userId: input.userId, + const now = input.now ?? new Date() + const updatedAt = now.toISOString() + const meter = userMeterRpc({ env: input.env, userId: input.userId }) + let result = await meter.read({ + resource: 'email_receives_per_day', day: input.day, + now: updatedAt, }) + if (result.outcome === 'needs_bootstrap') { + const baseline = await readCounter({ + db: input.db, + table: 'entitlement_daily_counters', + userId: input.userId, + day: input.day, + }) + await meter.initialize({ + resource: 'email_receives_per_day', + day: input.day, + count: baseline, + updatedAt, + }) + result = await meter.read({ + resource: 'email_receives_per_day', + day: input.day, + now: updatedAt, + }) + if (result.outcome === 'needs_bootstrap') { + throw new Error( + 'UserMeter inbound receive read still needs bootstrap after initialize.', + ) + } + } + return result.count } export async function readSystemInboundReceiveCount(input: { @@ -484,85 +519,101 @@ async function resolveConcurrentDelivery(input: { }) } +/** + * Charge one inbound receive via UserMeter, then persist the D1 delivery + * event. Delivery-id idempotency lives in the DO; D1 mirror is best-effort. + */ export async function chargeUserInboundDeliveryOnce(input: { db: D1Database + env: UserMeterEnv delivery: InboundDelivery plan: PlanName limit: number now: Date + waitUntil?: (promise: Promise) => void }): Promise { const existing = await resolveConcurrentDelivery(input) if (existing) return existing - const current = await readCounter({ - db: input.db, - table: 'entitlement_daily_counters', + + const updatedAt = input.now.toISOString() + const meter = userMeterRpc({ + env: input.env, userId: input.delivery.userId, + }) + let meterResult = await meter.consumeInboundDelivery({ + deliveryId: input.delivery.deliveryId, + resource: 'email_receives_per_day', day: input.delivery.quotaDay, + limit: input.limit, + updatedAt, }) - const throwLimitError = (): never => { + if (meterResult.outcome === 'needs_bootstrap') { + const baseline = await readCounter({ + db: input.db, + table: 'entitlement_daily_counters', + userId: input.delivery.userId, + day: input.delivery.quotaDay, + }) + await meter.initialize({ + resource: 'email_receives_per_day', + day: input.delivery.quotaDay, + count: baseline, + updatedAt, + }) + meterResult = await meter.consumeInboundDelivery({ + deliveryId: input.delivery.deliveryId, + resource: 'email_receives_per_day', + day: input.delivery.quotaDay, + limit: input.limit, + updatedAt, + }) + if (meterResult.outcome === 'needs_bootstrap') { + throw new Error( + 'UserMeter inbound delivery consume still needs bootstrap after initialize.', + ) + } + } + + const accepted = meterResult.consumed || meterResult.replayed + if (!accepted) { throw new EntitlementLimitError({ resource: 'email_receives_per_day', plan: input.plan, limit: input.limit, - current, + current: meterResult.count, upgradeHint: buildEntitlementUpgradeHint('email_receives_per_day'), }) } - if (input.limit < 1 || current >= input.limit) throwLimitError() - const operationTimestamp = randomOperationTimestamp(input.now) + + scheduleAbsoluteDailyEntitlementMirror({ + db: input.db, + userId: input.delivery.userId, + resource: meterResult.resource, + day: meterResult.day, + count: meterResult.count, + mirrorUpdatedAt: meterResult.mirrorUpdatedAt, + waitUntil: input.waitUntil, + }) + try { - const results = await input.db.batch([ - input.db - .prepare( - `INSERT INTO entitlement_daily_counters ( - user_id, resource, day, count, updated_at - ) VALUES (?, 'email_receives_per_day', ?, 1, ?) - ON CONFLICT(user_id, resource, day) DO UPDATE SET - count = entitlement_daily_counters.count + 1, - updated_at = excluded.updated_at - WHERE entitlement_daily_counters.count + 1 <= ? - AND NOT EXISTS ( - SELECT 1 FROM email_delivery_events - WHERE id = ? AND user_id = ? - )`, - ) - .bind( - input.delivery.userId, - input.delivery.quotaDay, - operationTimestamp, - input.limit, - input.delivery.deliveryId, - input.delivery.userId, - ), - input.db - .prepare( - `INSERT OR IGNORE INTO email_delivery_events ( - id, message_id, user_id, inbox_id, event_type, provider, - provider_event_id, detail_json, created_at - ) - SELECT ?, NULL, ?, ?, 'receive_started', ?, ?, ?, ? - WHERE EXISTS ( - SELECT 1 FROM entitlement_daily_counters - WHERE user_id = ? - AND resource = 'email_receives_per_day' - AND day = ? - AND updated_at = ? - )`, - ) - .bind( - input.delivery.deliveryId, - input.delivery.userId, - input.delivery.inboxId, - inboundProvider, - input.delivery.deliveryId, - JSON.stringify(input.delivery), - input.now.toISOString(), - input.delivery.userId, - input.delivery.quotaDay, - operationTimestamp, - ), - ]) - if (Number(results[1]?.meta.changes ?? 0) > 0) return input.delivery + const insert = await input.db + .prepare( + `INSERT OR IGNORE INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_event_id, detail_json, created_at + ) VALUES (?, NULL, ?, ?, 'receive_started', ?, ?, ?, ?)`, + ) + .bind( + input.delivery.deliveryId, + input.delivery.userId, + input.delivery.inboxId, + inboundProvider, + input.delivery.deliveryId, + JSON.stringify(input.delivery), + input.now.toISOString(), + ) + .run() + if (Number(insert.meta.changes ?? 0) > 0) return input.delivery } catch (error) { const committed = await resolveConcurrentDelivery(input).catch(() => null) if (committed) return committed @@ -570,7 +621,9 @@ export async function chargeUserInboundDeliveryOnce(input: { } const raced = await resolveConcurrentDelivery(input) if (raced) return raced - return throwLimitError() + throw new Error( + 'Inbound delivery charge was accepted but the delivery event was not persisted.', + ) } export async function chargeSystemInboundDeliveryOnce(input: { diff --git a/packages/worker/src/email/inbound-entitlements.workers.test.ts b/packages/worker/src/email/inbound-entitlements.workers.test.ts index b32a87ac98..41757b39cd 100644 --- a/packages/worker/src/email/inbound-entitlements.workers.test.ts +++ b/packages/worker/src/email/inbound-entitlements.workers.test.ts @@ -1,9 +1,23 @@ +import { runInDurableObject } from 'cloudflare:test' import { env } from 'cloudflare:workers' import { expect, test } from 'vitest' -import { maxPlanEmailLimits, planLimits } from '#worker/entitlements/plans.ts' import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { isEntitlementLimitError } from '#worker/entitlements/errors.ts' +import { maxPlanEmailLimits, planLimits } from '#worker/entitlements/plans.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { UserMeter } from '#worker/entitlements/user-meter-do.ts' +import { + createWaitUntilDrain, + withPatchedDbPrepare, +} from '#worker/test-support/user-meter.ts' import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { + buildInboundDelivery, + chargeUserInboundDeliveryOnce, + readUserInboundReceiveCount, +} from './inbound-delivery.ts' import { handleInboundEmail } from './inbound.ts' import { listEmailMessages } from './repo.ts' import { maxDetailedEmailRejectionEventsPerDay } from './service.ts' @@ -11,7 +25,6 @@ import { createForwardableEmailMessage } from './test-fixtures.ts' import { ensureEmailTestSchema } from './test-schema.ts' const platformBaseUrl = 'https://kody.example.com' -// User mail lives on the inbox. subdomain derived from APP_BASE_URL. const platformDomain = 'inbox.kody.example.com' function createInboundEnv() { @@ -40,7 +53,7 @@ async function seedAccountWithPlan(input: { stableUserId, ) .run() - return { username, address: `${username}@${platformDomain}` } + return { username, address: `${username}@${platformDomain}`, stableUserId } } function buildInboundMessage(address: string) { @@ -58,23 +71,45 @@ function buildInboundMessage(address: string) { }) } -async function setDailyReceiveCounter(userId: string, count: number) { +async function seedDailyReceiveCounter(userId: string, count: number) { + const day = utcDayKey() + const updatedAt = new Date().toISOString() + const stub = env.USER_METER.get( + env.USER_METER.idFromName(userMeterDurableObjectName(userId)), + ) + await runInDurableObject(stub, async (instance: UserMeter, state) => { + expect(instance).toBeInstanceOf(UserMeter) + await instance.read({ resource: 'email_receives_per_day', day }) + state.storage.sql.exec( + `INSERT INTO daily_counters (resource, day, count, revision, updated_at) + VALUES (?, ?, ?, 1, ?) + ON CONFLICT(resource, day) DO UPDATE SET + count = excluded.count, + revision = excluded.revision, + updated_at = excluded.updated_at`, + 'email_receives_per_day', + day, + count, + updatedAt, + ) + }) await env.APP_DB.prepare( `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) - VALUES (?, 'email_receives_per_day', ?, ?, ?)`, + VALUES (?, 'email_receives_per_day', ?, ?, ?) + ON CONFLICT(user_id, resource, day) DO UPDATE SET + count = excluded.count, + updated_at = excluded.updated_at`, ) - .bind(userId, utcDayKey(), count, new Date().toISOString()) + .bind(userId, day, count, updatedAt) .run() } async function readDailyReceiveCounter(userId: string) { - const row = await env.APP_DB.prepare( - `SELECT count FROM entitlement_daily_counters - WHERE user_id = ? AND resource = 'email_receives_per_day' AND day = ?`, - ) - .bind(userId, utcDayKey()) - .first<{ count: number }>() - return Number(row?.count ?? 0) + const result = await userMeterRpc({ env, userId }).read({ + resource: 'email_receives_per_day', + day: utcDayKey(), + }) + return result.outcome === 'ready' ? result.count : 0 } async function bulkInsertStoredMessages(userId: string, count: number) { @@ -141,6 +176,71 @@ async function readEmailReceivedRollup(userId: string) { .first<{ event_count: number; error_count: number; total_bytes: number }>() } +async function buildChargeableDelivery(input: { + userId: string + inboxId: string + address: string + raw?: string +}) { + const now = new Date() + const raw = + input.raw ?? + [ + 'From: Sender ', + `To: ${input.address}`, + 'Subject: Charge once', + `Message-ID: `, + '', + 'Body', + ].join('\r\n') + return await buildInboundDelivery({ + userId: input.userId, + inboxId: input.inboxId, + recipient: input.address, + envelopeFrom: 'sender@example.net', + rawMime: raw, + quotaDay: utcDayKey(now), + now, + }) +} + +function withFailingReceiveStartedInsert(input: { + deliveryId: string + message: string +}) { + let failNext = true + return withPatchedDbPrepare(env.APP_DB, (originalPrepare) => { + return ((query: string) => { + const statement = originalPrepare(query) + const isDeliveryInsert = + query.includes('INSERT OR IGNORE INTO email_delivery_events') && + query.includes("'receive_started'") + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: bound.first.bind(bound), + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: async () => { + if ( + isDeliveryInsert && + failNext && + params[0] === input.deliveryId + ) { + failNext = false + throw new Error(input.message) + } + return await bound.run() + }, + } + }, + } + }) as D1Database['prepare'] + }) +} + test('inbound email enforces free-plan receive, storage, and size limits then stores under quota', async () => { await ensureEmailTestSchema(env.APP_DB) await ensureUsageRollupsTestSchema(env.APP_DB) @@ -154,7 +254,7 @@ test('inbound email enforces free-plan receive, storage, and size limits then st email: receiveLimitEmail, plan: 'free', }) - await setDailyReceiveCounter(receiveLimitUserId, receiveLimit) + await seedDailyReceiveCounter(receiveLimitUserId, receiveLimit) const receiveLimitMessage = buildInboundMessage(receiveLimitAddress) await handleInboundEmail(receiveLimitMessage, createInboundEnv()) @@ -307,7 +407,7 @@ test('inbound email applies the max-plan email receive backstop', async () => { const userId = await createStableUserIdFromEmail(email) const { address } = await seedAccountWithPlan({ email, plan: 'max' }) const receiveLimit = maxPlanEmailLimits.email_receives_per_day - await setDailyReceiveCounter(userId, receiveLimit) + await seedDailyReceiveCounter(userId, receiveLimit) const message = buildInboundMessage(address) await handleInboundEmail(message, createInboundEnv()) @@ -328,7 +428,7 @@ test('inbound email stores at most five detailed rejection events per inbox per const limit = planLimits.free.maxEmailReceivesPerDay if (limit === null) throw new Error('Expected a numeric free limit.') const { address } = await seedAccountWithPlan({ email, plan: 'free' }) - await setDailyReceiveCounter(userId, limit) + await seedDailyReceiveCounter(userId, limit) const attempts = maxDetailedEmailRejectionEventsPerDay + 3 for (let index = 0; index < attempts; index += 1) { @@ -351,3 +451,459 @@ test('inbound email stores at most five detailed rejection events per inbox per error_count: attempts, }) }) + +test('inbound UserMeter delivery claims are idempotent, isolated, and over-limit exact', async () => { + await ensureEmailTestSchema(env.APP_DB) + const receiveLimit = planLimits.free.maxEmailReceivesPerDay + if (receiveLimit === null) throw new Error('Expected a numeric free limit.') + + const userAEmail = `meter-a-${crypto.randomUUID()}@example.com` + const userBEmail = `meter-b-${crypto.randomUUID()}@example.com` + const userAId = await createStableUserIdFromEmail(userAEmail) + const userBId = await createStableUserIdFromEmail(userBEmail) + const userA = await seedAccountWithPlan({ email: userAEmail, plan: 'free' }) + const userB = await seedAccountWithPlan({ email: userBEmail, plan: 'free' }) + const inboxA = `inbox-a-${crypto.randomUUID()}` + const inboxB = `inbox-b-${crypto.randomUUID()}` + const now = new Date() + const day = utcDayKey(now) + const drain = createWaitUntilDrain() + + const deliveryA = await buildChargeableDelivery({ + userId: userAId, + inboxId: inboxA, + address: userA.address, + }) + const deliveryARetry = { ...deliveryA } + const sharedDeliveryId = deliveryA.deliveryId + + const concurrent = await Promise.all([ + chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery: deliveryA, + plan: 'free', + limit: receiveLimit, + now, + waitUntil: drain.waitUntil, + }), + chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery: deliveryARetry, + plan: 'free', + limit: receiveLimit, + now, + waitUntil: drain.waitUntil, + }), + ]) + expect(concurrent[0]?.deliveryId).toBe(sharedDeliveryId) + expect(concurrent[1]?.deliveryId).toBe(sharedDeliveryId) + expect(await readDailyReceiveCounter(userAId)).toBe(1) + expect( + await env.APP_DB.prepare( + `SELECT COUNT(*) AS count FROM email_delivery_events + WHERE id = ? AND user_id = ?`, + ) + .bind(sharedDeliveryId, userAId) + .first<{ count: number }>(), + ).toEqual({ count: 1 }) + + const failingDelivery = await buildChargeableDelivery({ + userId: userAId, + inboxId: inboxA, + address: userA.address, + raw: [ + 'From: Sender ', + `To: ${userA.address}`, + 'Subject: Fail then retry', + `Message-ID: `, + '', + 'Retry body.', + ].join('\r\n'), + }) + using _patch = withFailingReceiveStartedInsert({ + deliveryId: failingDelivery.deliveryId, + message: 'simulated D1 delivery event insert failure', + }) + + await expect( + chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery: failingDelivery, + plan: 'free', + limit: receiveLimit, + now, + waitUntil: drain.waitUntil, + }), + ).rejects.toThrow('simulated D1 delivery event insert failure') + expect(await readDailyReceiveCounter(userAId)).toBe(2) + expect( + await env.APP_DB.prepare( + `SELECT id FROM email_delivery_events + WHERE id = ? AND user_id = ?`, + ) + .bind(failingDelivery.deliveryId, userAId) + .first(), + ).toBeNull() + + const recovered = await chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery: failingDelivery, + plan: 'free', + limit: receiveLimit, + now, + waitUntil: drain.waitUntil, + }) + expect(recovered.deliveryId).toBe(failingDelivery.deliveryId) + expect(await readDailyReceiveCounter(userAId)).toBe(2) + expect( + await env.APP_DB.prepare( + `SELECT id FROM email_delivery_events + WHERE id = ? AND user_id = ?`, + ) + .bind(failingDelivery.deliveryId, userAId) + .first(), + ).not.toBeNull() + + const deliveryB = await buildChargeableDelivery({ + userId: userBId, + inboxId: inboxB, + address: userB.address, + }) + await chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery: deliveryB, + plan: 'free', + limit: receiveLimit, + now, + waitUntil: drain.waitUntil, + }) + expect(await readDailyReceiveCounter(userAId)).toBe(2) + expect(await readDailyReceiveCounter(userBId)).toBe(1) + expect( + await readUserInboundReceiveCount({ + db: env.APP_DB, + env, + userId: userBId, + day, + now, + }), + ).toBe(1) + + const sharedClaimId = `email-inbound-delivery:shared-${crypto.randomUUID()}` + const meterA = userMeterRpc({ env, userId: userAId }) + const meterB = userMeterRpc({ env, userId: userBId }) + const claimA = await meterA.consumeInboundDelivery({ + deliveryId: sharedClaimId, + resource: 'email_receives_per_day', + day, + limit: receiveLimit, + updatedAt: now.toISOString(), + }) + const claimB = await meterB.consumeInboundDelivery({ + deliveryId: sharedClaimId, + resource: 'email_receives_per_day', + day, + limit: receiveLimit, + updatedAt: now.toISOString(), + }) + expect(claimA).toMatchObject({ + outcome: 'ready', + consumed: true, + replayed: false, + count: 3, + day, + resource: 'email_receives_per_day', + }) + expect(claimB).toMatchObject({ + outcome: 'ready', + consumed: true, + replayed: false, + count: 2, + day, + resource: 'email_receives_per_day', + }) + const replayA = await meterA.consumeInboundDelivery({ + deliveryId: sharedClaimId, + resource: 'email_receives_per_day', + day, + limit: receiveLimit, + updatedAt: now.toISOString(), + }) + expect(replayA).toMatchObject({ + outcome: 'ready', + consumed: false, + replayed: true, + count: 3, + day, + resource: 'email_receives_per_day', + }) + expect(await readDailyReceiveCounter(userAId)).toBe(3) + expect(await readDailyReceiveCounter(userBId)).toBe(2) + + await seedDailyReceiveCounter(userAId, receiveLimit) + const overLimitDelivery = await buildChargeableDelivery({ + userId: userAId, + inboxId: inboxA, + address: userA.address, + raw: [ + 'From: Sender ', + `To: ${userA.address}`, + 'Subject: Over limit', + `Message-ID: `, + '', + 'Denied.', + ].join('\r\n'), + }) + const denied = await chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery: overLimitDelivery, + plan: 'free', + limit: receiveLimit, + now, + waitUntil: drain.waitUntil, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(isEntitlementLimitError(denied)).toBe(true) + expect(denied).toMatchObject({ + details: { + code: 'entitlement_limit_exceeded', + resource: 'email_receives_per_day', + plan: 'free', + limit: receiveLimit, + current: receiveLimit, + }, + }) + expect(await readDailyReceiveCounter(userAId)).toBe(receiveLimit) + await drain.drain() +}, 30_000) + +test('inbound UserMeter claim+consume rolls back together when claim insert fails', async () => { + await ensureEmailTestSchema(env.APP_DB) + const email = `meter-atomic-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(email) + await seedAccountWithPlan({ email, plan: 'free' }) + const day = '2026-07-30' + const updatedAt = '2026-07-30T12:00:00.000Z' + const deliveryId = `email-inbound-delivery:atomic-${crypto.randomUUID()}` + const stub = env.USER_METER.get( + env.USER_METER.idFromName(userMeterDurableObjectName(userId)), + ) + + await runInDurableObject(stub, async (instance: UserMeter, state) => { + expect(instance).toBeInstanceOf(UserMeter) + await instance.initialize({ + resource: 'email_receives_per_day', + day, + count: 0, + updatedAt, + }) + + const sql = state.storage.sql + const originalExec = sql.exec.bind(sql) + sql.exec = ((query: string, ...bindings: Array) => { + if ( + typeof query === 'string' && + query.includes('INSERT INTO inbound_delivery_claims') + ) { + throw new Error('injected inbound claim insert failure') + } + return originalExec(query, ...bindings) + }) as typeof sql.exec + + const receiveLimit = planLimits.free.maxEmailReceivesPerDay + if (receiveLimit === null) throw new Error('Expected a numeric free limit.') + + await expect( + instance.consumeInboundDelivery({ + deliveryId, + resource: 'email_receives_per_day', + day, + limit: receiveLimit, + updatedAt, + }), + ).rejects.toThrow('injected inbound claim insert failure') + + sql.exec = originalExec + + expect( + await instance.read({ resource: 'email_receives_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 0 }) + expect( + state.storage.sql + .exec<{ n: number }>( + `SELECT COUNT(*) AS n FROM inbound_delivery_claims + WHERE delivery_id = ?`, + deliveryId, + ) + .toArray()[0]?.n, + ).toBe(0) + + const accepted = await instance.consumeInboundDelivery({ + deliveryId, + resource: 'email_receives_per_day', + day, + limit: receiveLimit, + updatedAt, + }) + expect(accepted).toMatchObject({ + outcome: 'ready', + consumed: true, + replayed: false, + count: 1, + day, + resource: 'email_receives_per_day', + }) + expect( + state.storage.sql + .exec<{ n: number }>( + `SELECT COUNT(*) AS n FROM inbound_delivery_claims + WHERE delivery_id = ?`, + deliveryId, + ) + .toArray()[0]?.n, + ).toBe(1) + + const foreignDeliveryId = `email-inbound-delivery:foreign-${crypto.randomUUID()}` + state.storage.sql.exec( + `INSERT INTO inbound_delivery_claims ( + delivery_id, resource, day, count_after, revision, claimed_at + ) VALUES (?, 'email_sends_per_day', ?, 1, 1, ?)`, + foreignDeliveryId, + day, + updatedAt, + ) + await expect( + instance.consumeInboundDelivery({ + deliveryId: foreignDeliveryId, + resource: 'email_receives_per_day', + day, + limit: receiveLimit, + updatedAt, + }), + ).rejects.toThrow(/was claimed for "email_sends_per_day"/) + }) +}) + +test('inbound UserMeter replay keeps original claim day across UTC midnight', async () => { + await ensureEmailTestSchema(env.APP_DB) + const receiveLimit = planLimits.free.maxEmailReceivesPerDay + if (receiveLimit === null) throw new Error('Expected a numeric free limit.') + + const email = `meter-cross-day-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(email) + const account = await seedAccountWithPlan({ email, plan: 'free' }) + const inboxId = `inbox-cross-${crypto.randomUUID()}` + const dayN = '2026-07-30' + const dayN1 = '2026-07-31' + const drain = createWaitUntilDrain() + const meter = userMeterRpc({ env, userId }) + + await meter.initialize({ + resource: 'email_receives_per_day', + day: dayN, + count: 0, + updatedAt: `${dayN}T23:50:00.000Z`, + }) + + const delivery = await buildInboundDelivery({ + userId, + inboxId, + recipient: account.address, + envelopeFrom: 'sender@example.net', + rawMime: [ + 'From: Sender ', + `To: ${account.address}`, + 'Subject: Cross-day retry', + `Message-ID: `, + '', + 'Body', + ].join('\r\n'), + quotaDay: dayN, + now: new Date(`${dayN}T23:59:00.000Z`), + }) + + using _patch = withFailingReceiveStartedInsert({ + deliveryId: delivery.deliveryId, + message: 'simulated cross-day D1 insert failure', + }) + + await expect( + chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery, + plan: 'free', + limit: receiveLimit, + now: new Date(`${dayN}T23:59:30.000Z`), + waitUntil: drain.waitUntil, + }), + ).rejects.toThrow('simulated cross-day D1 insert failure') + + expect( + await meter.read({ resource: 'email_receives_per_day', day: dayN }), + ).toMatchObject({ outcome: 'ready', count: 1 }) + expect( + await meter.read({ resource: 'email_receives_per_day', day: dayN1 }), + ).toMatchObject({ outcome: 'needs_bootstrap' }) + + const replay = await meter.consumeInboundDelivery({ + deliveryId: delivery.deliveryId, + resource: 'email_receives_per_day', + day: dayN1, + limit: receiveLimit, + updatedAt: `${dayN1}T00:01:00.000Z`, + }) + expect(replay).toMatchObject({ + outcome: 'ready', + consumed: false, + replayed: true, + count: 1, + day: dayN, + resource: 'email_receives_per_day', + }) + + const recovered = await chargeUserInboundDeliveryOnce({ + db: env.APP_DB, + env, + delivery: { + ...delivery, + quotaDay: dayN1, + }, + plan: 'free', + limit: receiveLimit, + now: new Date(`${dayN1}T00:01:00.000Z`), + waitUntil: drain.waitUntil, + }) + expect(recovered.deliveryId).toBe(delivery.deliveryId) + await drain.drain() + + expect( + await meter.read({ resource: 'email_receives_per_day', day: dayN }), + ).toMatchObject({ outcome: 'ready', count: 1 }) + expect( + await meter.read({ resource: 'email_receives_per_day', day: dayN1 }), + ).toMatchObject({ outcome: 'needs_bootstrap' }) + + const mirroredDayN = await env.APP_DB.prepare( + `SELECT count FROM entitlement_daily_counters + WHERE user_id = ? AND resource = 'email_receives_per_day' AND day = ?`, + ) + .bind(userId, dayN) + .first<{ count: number }>() + const mirroredDayN1 = await env.APP_DB.prepare( + `SELECT count FROM entitlement_daily_counters + WHERE user_id = ? AND resource = 'email_receives_per_day' AND day = ?`, + ) + .bind(userId, dayN1) + .first<{ count: number }>() + expect(Number(mirroredDayN?.count ?? 0)).toBe(1) + expect(mirroredDayN1).toBeNull() +}, 30_000) diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index 31e24e5fa5..4f5a72c58e 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -196,6 +196,7 @@ export async function handleInboundEmail( | 'APP_BASE_URL' | 'USER_EMAIL_DOMAIN' | 'USAGE_EVENTS' + | 'USER_METER' >, ctx?: ExecutionContext, ) { @@ -482,8 +483,10 @@ export async function handleInboundEmail( ) const receivesToday = await readUserInboundReceiveCount({ db: env.APP_DB, + env, userId, day: userInboundQuotaDay(quotaNow), + now: quotaNow, }) if (receivesToday >= receiveLimit) { message.setReject('Recipient mailbox is over quota.') @@ -530,12 +533,16 @@ export async function handleInboundEmail( }) } } + const waitUntil = ctx + ? (promise: Promise) => ctx.waitUntil(promise) + : undefined let claimedDelivery: InboundDelivery try { claimedDelivery = existingDelivery ?? (await chargeUserInboundDeliveryOnce({ db: env.APP_DB, + env, delivery: { ...delivery, quotaDay: userInboundQuotaDay(quotaNow), @@ -546,6 +553,7 @@ export async function handleInboundEmail( 'email_receives_per_day', ), now: quotaNow, + waitUntil, })) } catch (error) { if (!isEntitlementLimitError(error)) throw error diff --git a/packages/worker/src/email/inbound.workers.test.ts b/packages/worker/src/email/inbound.workers.test.ts index eeda94dd88..7d9ff38520 100644 --- a/packages/worker/src/email/inbound.workers.test.ts +++ b/packages/worker/src/email/inbound.workers.test.ts @@ -1,5 +1,6 @@ import { env } from 'cloudflare:workers' import { expect, test, vi } from 'vitest' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' import { handleInboundEmail } from './inbound.ts' import { processInboundDeliveryEffects } from './inbound-effects.ts' import { @@ -848,13 +849,18 @@ test('inbound email stores raw MIME in R2 and readers and deletes follow the blo }) async function readUserDailyReceiveCount(userId: string) { - const row = await env.APP_DB.prepare( - `SELECT count FROM entitlement_daily_counters - WHERE user_id = ? AND resource = 'email_receives_per_day' AND day = ?`, - ) - .bind(userId, new Date().toISOString().slice(0, 10)) - .first<{ count: number }>() - return Number(row?.count ?? 0) + const result = await userMeterRpc({ env, userId }).read({ + resource: 'email_receives_per_day', + day: new Date().toISOString().slice(0, 10), + }) + return result.outcome === 'ready' ? result.count : 0 +} + +async function readUserTotalReceiveCount(userId: string) { + const exported = await userMeterRpc({ env, userId }).exportCounters({}) + return exported.counters + .filter((row) => row.resource === 'email_receives_per_day') + .reduce((total, row) => total + row.count, 0) } function createFailingEmailBlobs() { @@ -1093,7 +1099,7 @@ test('charged retry repairs after unrelated writes fill storage bytes', async () ).toHaveLength(1) }) -test('ambiguous quota-ledger batch response charges one delivery exactly once', async () => { +test('ambiguous delivery-event insert response charges one delivery exactly once', async () => { silenceIncidentalRuntimeWarnings() await ensureEmailTestSchema(env.APP_DB) const username = `quota-ambiguous-${crypto.randomUUID().slice(0, 8)}` @@ -1105,19 +1111,37 @@ test('ambiguous quota-ledger batch response charges one delivery exactly once', email: accountEmail, username, }) - let batchCalls = 0 + let loseNextDeliveryInsertResponse = true const ambiguousDb = new Proxy(env.APP_DB, { get(target, property, receiver) { - if (property === 'batch') { - return async (statements: Parameters[0]) => { - batchCalls++ - const result = await target.batch(statements) - // The account-write lease acquire is the first batch. Lose the - // response from the following quota-ledger batch. - if (batchCalls === 2) { - throw new Error('simulated quota batch response loss') + if (property === 'prepare') { + return (query: string) => { + const statement = target.prepare(query) + const isDeliveryInsert = + query.includes('INSERT OR IGNORE INTO email_delivery_events') && + query.includes("'receive_started'") + if (!isDeliveryInsert) return statement + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: bound.first.bind(bound), + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: async () => { + const result = await bound.run() + if (loseNextDeliveryInsertResponse) { + loseNextDeliveryInsertResponse = false + throw new Error( + 'simulated delivery event insert response loss', + ) + } + return result + }, + } + }, } - return result } } const value = Reflect.get(target, property, receiver) @@ -1391,13 +1415,7 @@ test('byte-identical mail dedupes only inside the explicit delivery window', asy limit: 10, }), ).toHaveLength(2) - const counter = await env.APP_DB.prepare( - `SELECT SUM(count) AS count FROM entitlement_daily_counters - WHERE user_id = ? AND resource = 'email_receives_per_day'`, - ) - .bind(userId) - .first<{ count: number }>() - expect(Number(counter?.count ?? 0)).toBe(2) + expect(await readUserTotalReceiveCount(userId)).toBe(2) expect( await env.APP_DB.prepare( `SELECT event_count FROM usage_rollups diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index 64e94ef2d5..62700cf7e9 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -43,6 +43,7 @@ type SendEmailEnv = Pick< | 'CLOUDFLARE_ACCOUNT_ID' | 'CLOUDFLARE_API_BASE_URL' | 'CLOUDFLARE_API_TOKEN' + | 'USER_METER' > /** @@ -550,11 +551,11 @@ export async function sendOutboundEmail( }) + attachmentBytesTotal, }) - // Atomic check-and-increment: the counter tracks attempts for every user - // and denies the send when a plan's daily limit is reached. The - // `max` plan uses finite email caps rather than uncapped mail. + // Atomic check-and-increment via UserMeter. No ExecutionContext here; + // the legacy D1 mirror uses the service's caught best-effort fallback. await consumeDailyEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: sender.accountEmail, resource: 'email_sends_per_day', diff --git a/packages/worker/src/email/outbound.workers.test.ts b/packages/worker/src/email/outbound.workers.test.ts index 3b6742595c..3f87a19883 100644 --- a/packages/worker/src/email/outbound.workers.test.ts +++ b/packages/worker/src/email/outbound.workers.test.ts @@ -1,4 +1,5 @@ import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' import { expect, test } from 'vitest' import { http, HttpResponse } from 'msw' import { bytesToBase64 } from '@kody-internal/shared/base64.ts' @@ -22,7 +23,9 @@ import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidenta import { createMswNodeServer } from '#worker/test-support/msw-node-server.ts' import { isEntitlementLimitError } from '#worker/entitlements/errors.ts' import { maxPlanEmailLimits, planLimits } from '#worker/entitlements/plans.ts' -import { incrementDailyEntitlementCounter } from '#worker/entitlements/service.ts' +import { UserMeter } from '#worker/entitlements/user-meter-do.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' import { utcDayKey } from '@kody-internal/shared/date-keys.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' @@ -70,13 +73,35 @@ async function seedVerifiedAccount(input: { } async function readDailyEmailSendCounter(userId: string) { - const row = await env.APP_DB.prepare( - `SELECT count FROM entitlement_daily_counters - WHERE user_id = ? AND resource = ? AND day = ?`, + const result = await userMeterRpc({ env, userId }).read({ + resource: 'email_sends_per_day', + day: utcDayKey(), + }) + return result.outcome === 'ready' ? result.count : 0 +} + +async function seedDailyEmailSendCounter(userId: string, count: number) { + const day = utcDayKey() + const updatedAt = new Date().toISOString() + const stub = env.USER_METER.get( + env.USER_METER.idFromName(userMeterDurableObjectName(userId)), ) - .bind(userId, 'email_sends_per_day', utcDayKey()) - .first<{ count: number }>() - return Number(row?.count ?? 0) + await runInDurableObject(stub, async (instance: UserMeter, state) => { + expect(instance).toBeInstanceOf(UserMeter) + await instance.read({ resource: 'email_sends_per_day', day }) + state.storage.sql.exec( + `INSERT INTO daily_counters (resource, day, count, revision, updated_at) + VALUES (?, ?, ?, 1, ?) + ON CONFLICT(resource, day) DO UPDATE SET + count = excluded.count, + revision = excluded.revision, + updated_at = excluded.updated_at`, + 'email_sends_per_day', + day, + count, + updatedAt, + ) + }) } async function sendSelfNotification(input: { @@ -187,7 +212,7 @@ test('sendOutboundEmail sends from the platform-assigned username address to the limit: 5, }) expect(listed.map((message) => message.id)).toContain(result.message.id) -}) +}, 30_000) test('sendOutboundEmail rejects suspended accounts', async () => { silenceIncidentalRuntimeWarnings() @@ -823,7 +848,7 @@ test('sendOutboundEmail rejects invalid and oversized attachments', async () => limit: 5, }), ).toEqual([]) -}) +}, 30_000) test('sendOutboundEmail enforces email_sends_per_day for plan users at the limit', async () => { await ensureEmailTestSchema(env.APP_DB) @@ -832,18 +857,7 @@ test('sendOutboundEmail enforces email_sends_per_day for plan users at the limit const limit = planLimits.pro.maxEmailSendsPerDay if (limit === null) throw new Error('Expected a numeric pro email limit.') await seedVerifiedAccount({ email, plan: 'pro' }) - await env.APP_DB.prepare( - `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) - VALUES (?, ?, ?, ?, ?)`, - ) - .bind( - userId, - 'email_sends_per_day', - utcDayKey(), - limit, - new Date().toISOString(), - ) - .run() + await seedDailyEmailSendCounter(userId, limit) const error = await sendSelfNotification({ userId, @@ -863,7 +877,7 @@ test('sendOutboundEmail enforces email_sends_per_day for plan users at the limit current: limit, }) expect(error.message).toContain(`at most ${limit} email sends per day`) -}) +}, 30_000) test('sendOutboundEmail binds plan limits even when the caller context has no account email', async () => { await ensureEmailTestSchema(env.APP_DB) @@ -872,18 +886,7 @@ test('sendOutboundEmail binds plan limits even when the caller context has no ac const limit = planLimits.pro.maxEmailSendsPerDay if (limit === null) throw new Error('Expected a numeric pro email limit.') await seedVerifiedAccount({ email, plan: 'pro' }) - await env.APP_DB.prepare( - `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) - VALUES (?, ?, ?, ?, ?)`, - ) - .bind( - userId, - 'email_sends_per_day', - utcDayKey(), - limit, - new Date().toISOString(), - ) - .run() + await seedDailyEmailSendCounter(userId, limit) // Package subscription contexts pass an empty account email; the plan // limit must still apply (no fallback-limit bypass). @@ -906,7 +909,7 @@ test('sendOutboundEmail binds plan limits even when the caller context has no ac plan: 'pro', limit, }) -}) +}, 30_000) test('sendOutboundEmail increments the daily counter when under the plan limit', async () => { await ensureEmailTestSchema(env.APP_DB) @@ -922,7 +925,7 @@ test('sendOutboundEmail increments the daily counter when under the plan limit', expect(result.status).toBe('sent') expect(await readDailyEmailSendCounter(userId)).toBe(1) -}) +}, 30_000) test('sendOutboundEmail caps max-plan users at the email daily backstop', async () => { await ensureEmailTestSchema(env.APP_DB) @@ -930,17 +933,10 @@ test('sendOutboundEmail caps max-plan users at the email daily backstop', async const email = `max-${crypto.randomUUID()}@example.com` const userId = await createStableUserIdFromEmail(email) await seedVerifiedAccount({ email, plan: 'max' }) - for ( - let index = 0; - index < maxPlanEmailLimits.email_sends_per_day - 1; - index += 1 - ) { - await incrementDailyEntitlementCounter({ - db: env.APP_DB, - userId, - resource: 'email_sends_per_day', - }) - } + await seedDailyEmailSendCounter( + userId, + maxPlanEmailLimits.email_sends_per_day - 1, + ) // One send left under the backstop succeeds... const result = await sendSelfNotification({ userId, accountEmail: email }) expect(result.status).toBe('sent') @@ -969,4 +965,4 @@ test('sendOutboundEmail caps max-plan users at the email daily backstop', async expect(await readDailyEmailSendCounter(userId)).toBe( maxPlanEmailLimits.email_sends_per_day, ) -}) +}, 30_000) diff --git a/packages/worker/src/entitlements/entitlements.node.test.ts b/packages/worker/src/entitlements/entitlements.node.test.ts index 2b3585ce6b..568a352fdc 100644 --- a/packages/worker/src/entitlements/entitlements.node.test.ts +++ b/packages/worker/src/entitlements/entitlements.node.test.ts @@ -34,12 +34,17 @@ import { refundDailyEntitlement, } from './service.ts' import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { + createInMemoryUserMeterEnv, + createWaitUntilDrain, +} from '#worker/test-support/user-meter.ts' type CounterRow = { user_id: string resource: string day: string count: number + updated_at?: string } function createEntitlementsTestDb( @@ -207,8 +212,12 @@ function createEntitlementsTestDb( return { meta: { changes: 1 } } } if (query.includes('INSERT INTO entitlement_daily_counters')) { + const isAbsoluteMirror = + query.includes('count = excluded.count') && + query.includes('updated_at < excluded.updated_at') const isConditionalConsume = query.includes('count + 1 <= ?') const amount = isConditionalConsume ? 1 : Number(params[3]) + const mirrorUpdatedAt = String(params[4] ?? '') const existing = counters.find( (counter) => counter.user_id === params[0] && @@ -222,13 +231,28 @@ function createEntitlementsTestDb( ) { return { meta: { changes: 0 } } } - existing.count += amount + if (isAbsoluteMirror) { + const existingUpdatedAt = existing.updated_at ?? '' + if ( + existingUpdatedAt !== '' && + !(existingUpdatedAt < mirrorUpdatedAt) + ) { + return { meta: { changes: 0 } } + } + existing.count = amount + existing.updated_at = mirrorUpdatedAt + } else { + existing.count += amount + } } else { counters.push({ user_id: String(params[0]), resource: String(params[1]), day: String(params[2]), count: amount, + ...(isAbsoluteMirror + ? { updated_at: mirrorUpdatedAt } + : {}), }) } return { meta: { changes: 1 } } @@ -752,31 +776,31 @@ test('plan user daily entitlements increment, enforce at limit, and reset on a n const { db, counters } = createEntitlementsTestDb({ users: [{ email: plannedEmail, plan: 'free', stable_user_id: userId }], }) + const { env } = createInMemoryUserMeterEnv() + const mirror = createWaitUntilDrain() expect(utcDayKey(now)).toBe('2026-07-05') const limit = planLimits.free.maxEmailSendsPerDay if (limit === null) throw new Error('Expected a numeric email send limit.') for (let index = 0; index < limit; index += 1) { - await assertWithinEntitlement({ + await consumeDailyEntitlement({ db, + env, userId, email: plannedEmail, resource: 'email_sends_per_day', now, - }) - await incrementDailyEntitlementCounter({ - db, - userId, - resource: 'email_sends_per_day', - now, + waitUntil: mirror.waitUntil, }) } + await mirror.drain() expect(counters).toEqual([ { user_id: userId, resource: 'email_sends_per_day', day: '2026-07-05', count: limit, + updated_at: expect.stringMatching(/^r\/0+[0-9]+$/), }, ]) await expect( @@ -791,10 +815,12 @@ test('plan user daily entitlements increment, enforce at limit, and reset on a n const denied = await consumeDailyEntitlement({ db, + env, userId, email: plannedEmail, resource: 'email_sends_per_day', now, + waitUntil: mirror.waitUntil, }).then( () => null, (thrown: unknown) => thrown, @@ -812,47 +838,57 @@ test('plan user daily entitlements increment, enforce at limit, and reset on a n expect(counters[0]?.count).toBe(limit) const nextDay = new Date('2026-07-06T00:00:01.000Z') - await assertWithinEntitlement({ - db, - userId, - email: plannedEmail, - resource: 'email_sends_per_day', - now: nextDay, - }) await consumeDailyEntitlement({ db, + env, userId, email: plannedEmail, resource: 'email_sends_per_day', now: nextDay, + waitUntil: mirror.waitUntil, }) + await mirror.drain() expect(counters).toHaveLength(2) expect(counters[1]?.count).toBe(1) }) test('refundDailyEntitlement decrements the user/day counter and floors at zero', async () => { const { db, counters } = createEntitlementsTestDb() + const { env } = createInMemoryUserMeterEnv() + const mirror = createWaitUntilDrain() const now = new Date('2026-07-05T15:00:00.000Z') - await incrementDailyEntitlementCounter({ - db, - userId: 'user-1', - resource: 'email_receives_per_day', - amount: 2, - now, - }) - await incrementDailyEntitlementCounter({ - db, - userId: 'user-2', - resource: 'email_receives_per_day', - amount: 3, - now, - }) + for (let index = 0; index < 2; index += 1) { + await consumeDailyEntitlement({ + db, + env, + userId: 'user-1', + email: null, + resource: 'email_receives_per_day', + now, + waitUntil: mirror.waitUntil, + }) + } + for (let index = 0; index < 3; index += 1) { + await consumeDailyEntitlement({ + db, + env, + userId: 'user-2', + email: null, + resource: 'email_receives_per_day', + now, + waitUntil: mirror.waitUntil, + }) + } + await mirror.drain() await refundDailyEntitlement({ db, + env, userId: 'user-1', resource: 'email_receives_per_day', now, + waitUntil: mirror.waitUntil, }) + await mirror.drain() expect( counters.find( (row) => @@ -868,16 +904,21 @@ test('refundDailyEntitlement decrements the user/day counter and floors at zero' await refundDailyEntitlement({ db, + env, userId: 'user-1', resource: 'email_receives_per_day', now, + waitUntil: mirror.waitUntil, }) await refundDailyEntitlement({ db, + env, userId: 'user-1', resource: 'email_receives_per_day', now, + waitUntil: mirror.waitUntil, }) + await mirror.drain() expect( counters.find( (row) => @@ -888,25 +929,32 @@ test('refundDailyEntitlement decrements the user/day counter and floors at zero' test('missing-email lookups fail closed and honor free email caps', async () => { const { db, counters } = createEntitlementsTestDb() + const { env } = createInMemoryUserMeterEnv() + const mirror = createWaitUntilDrain() const sendLimit = planLimits.free.maxEmailSendsPerDay const now = new Date('2026-07-05T15:00:00.000Z') for (let index = 0; index < sendLimit; index += 1) { await consumeDailyEntitlement({ db, + env, userId: 'user-1', email: null, resource: 'email_sends_per_day', now, + waitUntil: mirror.waitUntil, }) } + await mirror.drain() expect(counters[0]?.count).toBe(sendLimit) await expect( consumeDailyEntitlement({ db, + env, userId: 'user-1', email: null, resource: 'email_sends_per_day', now, + waitUntil: mirror.waitUntil, }), ).rejects.toBeInstanceOf(EntitlementLimitError) @@ -914,22 +962,27 @@ test('missing-email lookups fail closed and honor free email caps', async () => for (let index = 0; index < receiveLimit; index += 1) { await consumeDailyEntitlement({ db, + env, userId: 'user-1', email: null, resource: 'email_receives_per_day', now, + waitUntil: mirror.waitUntil, }) } + await mirror.drain() expect( counters.find((row) => row.resource === 'email_receives_per_day')?.count, ).toBe(receiveLimit) const denied = await consumeDailyEntitlement({ db, + env, userId: 'user-1', email: null, resource: 'email_receives_per_day', now, + waitUntil: mirror.waitUntil, }).then( () => null, (thrown: unknown) => thrown, diff --git a/packages/worker/src/entitlements/service.ts b/packages/worker/src/entitlements/service.ts index c7a7f534a1..3eafe6b86a 100644 --- a/packages/worker/src/entitlements/service.ts +++ b/packages/worker/src/entitlements/service.ts @@ -9,6 +9,11 @@ import { type EntitlementResource, type PlanName, } from './plans.ts' +import { + isDailyEntitlementResource, + type DailyEntitlementResource, +} from './user-meter-do.ts' +import { userMeterRpc, type UserMeterEnv } from './user-meter-client.ts' const stableUserIdPattern = /^[a-f0-9]{64}$/i @@ -193,9 +198,9 @@ export async function findUserAccountByStableUserId( } /** - * Increment a daily counter for a rate-style entitlement (for example - * email sends per day). Counters accumulate for every user regardless of - * plan so that assigning a plan later enforces against real usage. + * Legacy D1 mirror write for a daily entitlement counter. Expand-phase + * enforcement is authoritative in UserMeter; this helper remains for tests, + * backfills, and the best-effort mirror scheduled after DO consume/refund. */ export async function incrementDailyEntitlementCounter(input: { db: D1Database @@ -223,6 +228,110 @@ export async function incrementDailyEntitlementCounter(input: { .run() } +function assertDailyEntitlementResource( + resource: EntitlementResource, +): DailyEntitlementResource { + if (!isDailyEntitlementResource(resource)) { + throw new Error( + `Expected a daily entitlement resource; got ${JSON.stringify(resource)}.`, + ) + } + return resource +} + +// Best-effort mirror scheduling: prefer `waitUntil` when available; otherwise +// catch so the promise cannot surface as unhandled. Never awaited by callers. +function scheduleDailyEntitlementMirror( + work: Promise, + waitUntil?: (promise: Promise) => void, +) { + const tracked = work.catch((error: unknown) => { + console.warn('entitlement-daily-mirror-failed', error) + }) + if (waitUntil) { + waitUntil(tracked) + return + } + void tracked +} + +// Absolute D1 mirror ordered by DO-minted `mirrorUpdatedAt` (revision-primary) +// so a late older write cannot overwrite newer state, including refunds. +async function mirrorDailyEntitlementAbsoluteCount(input: { + db: D1Database + userId: string + resource: DailyEntitlementResource + day: string + count: number + mirrorUpdatedAt: string +}) { + await input.db + .prepare( + `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(user_id, resource, day) DO UPDATE SET + count = excluded.count, + updated_at = excluded.updated_at + WHERE entitlement_daily_counters.updated_at < excluded.updated_at`, + ) + .bind( + input.userId, + input.resource, + input.day, + input.count, + input.mirrorUpdatedAt, + ) + .run() +} + +/** Best-effort absolute D1 mirror of UserMeter daily counter state. */ +export function scheduleAbsoluteDailyEntitlementMirror(input: { + db: D1Database + userId: string + resource: EntitlementResource + day: string + count: number + mirrorUpdatedAt: string + waitUntil?: (promise: Promise) => void +}): void { + const resource = assertDailyEntitlementResource(input.resource) + scheduleDailyEntitlementMirror( + mirrorDailyEntitlementAbsoluteCount({ + db: input.db, + userId: input.userId, + resource, + day: input.day, + count: input.count, + mirrorUpdatedAt: input.mirrorUpdatedAt, + }), + input.waitUntil, + ) +} + +async function ensureUserMeterCounterInitialized(input: { + db: D1Database + env: UserMeterEnv + userId: string + resource: DailyEntitlementResource + day: string + updatedAt: string + now: Date +}) { + const baseline = await readDailyEntitlementCounter({ + db: input.db, + userId: input.userId, + resource: input.resource, + now: input.now, + }) + const meter = userMeterRpc({ env: input.env, userId: input.userId }) + await meter.initialize({ + resource: input.resource, + day: input.day, + count: baseline, + updatedAt: input.updatedAt, + }) +} + async function readDailyEntitlementCounter(input: { db: D1Database userId: string @@ -239,6 +348,52 @@ async function readDailyEntitlementCounter(input: { return Number(row?.count ?? 0) } +/** + * Point-read one daily entitlement counter from UserMeter. Cold meters + * bootstrap once from the legacy D1 point row, then re-read; warm meters + * return the DO count without touching D1. + */ +export async function readDailyEntitlementResourceUsage(input: { + db: D1Database + env: UserMeterEnv + userId: string + resource: EntitlementResource + now?: Date +}): Promise { + const resource = assertDailyEntitlementResource(input.resource) + const now = input.now ?? new Date() + const day = utcDayKey(now) + const updatedAt = now.toISOString() + const meter = userMeterRpc({ env: input.env, userId: input.userId }) + let result = await meter.read({ + resource, + day, + now: updatedAt, + }) + if (result.outcome === 'needs_bootstrap') { + await ensureUserMeterCounterInitialized({ + db: input.db, + env: input.env, + userId: input.userId, + resource, + day, + updatedAt, + now, + }) + result = await meter.read({ + resource, + day, + now: updatedAt, + }) + if (result.outcome === 'needs_bootstrap') { + throw new Error( + 'UserMeter daily entitlement read still needs bootstrap after initialize.', + ) + } + } + return result.count +} + async function countRows(db: D1Database, sql: string, params: Array) { const row = await db .prepare(sql) @@ -730,6 +885,34 @@ export async function readEntitlementResourceUsage(input: { } } +/** + * Authoritative current usage for any entitlement resource: daily counters + * via UserMeter, everything else via the legacy D1 helpers. + */ +export async function readCurrentEntitlementResourceUsage(input: { + db: D1Database + env: UserMeterEnv + userId: string + resource: EntitlementResource + now: Date +}): Promise { + if (isDailyEntitlementResource(input.resource)) { + return await readDailyEntitlementResourceUsage({ + db: input.db, + env: input.env, + userId: input.userId, + resource: input.resource, + now: input.now, + }) + } + return await readEntitlementResourceUsage({ + db: input.db, + userId: input.userId, + resource: input.resource, + now: input.now, + }) +} + export async function assertWithinStorageBytesEntitlement(input: { db: D1Database userId: string @@ -841,95 +1024,128 @@ export async function assertWithinEntitlement( } } -/** - * Atomically consume one unit of a daily rate-style entitlement (check and - * increment in a single conditional D1 upsert), throwing - * EntitlementLimitError when the consumption would exceed the plan limit. - * This avoids the check-then-increment race that separate - * assertWithinEntitlement + incrementDailyEntitlementCounter calls would - * have under concurrent requests, and evaluates the UTC day key once. - * Every resolved plan has finite numeric limits. - */ -export async function consumeDailyEntitlement(input: { +export type ConsumeDailyEntitlementInput = { db: D1Database + /** Must expose `USER_METER` (authoritative); D1 is a best-effort mirror. */ + env: UserMeterEnv userId: string email: string | null | undefined resource: EntitlementResource now?: Date -}): Promise { + /** Prefer `ctx.waitUntil` for the legacy D1 mirror; never awaited here. */ + waitUntil?: (promise: Promise) => void +} + +/** + * Atomically consume one daily entitlement unit via UserMeter, throwing + * EntitlementLimitError when the plan limit would be exceeded. Cold keys + * bootstrap once from legacy D1; warm path awaits only the DO RPC. Mirror + * writes are best-effort and cannot affect enforcement. + */ +export async function consumeDailyEntitlement( + input: ConsumeDailyEntitlementInput, +): Promise { + const resource = assertDailyEntitlementResource(input.resource) const now = input.now ?? new Date() + const day = utcDayKey(now) + const updatedAt = now.toISOString() // Cached plan resolution: this runs on every execute call and every - // sandbox outbound fetch, and the atomic counter upsert below is the only - // part that must see fresh state. + // sandbox outbound fetch; the UserMeter consume below is the only + // counter state that must be fresh. const plan = await getCachedUserPlan(input.db, { userId: input.userId, email: input.email, }) - const limit = resolvePlanLimit(plan, input.resource) - const throwLimitError = async () => { - throw new EntitlementLimitError({ - resource: input.resource, - plan, + const limit = resolvePlanLimit(plan, resource) + const meter = userMeterRpc({ env: input.env, userId: input.userId }) + let result = await meter.consume({ + resource, + day, + limit, + updatedAt, + }) + if (result.outcome === 'needs_bootstrap') { + await ensureUserMeterCounterInitialized({ + db: input.db, + env: input.env, + userId: input.userId, + resource, + day, + updatedAt, + now, + }) + result = await meter.consume({ + resource, + day, limit, - current: await readDailyEntitlementCounter({ - db: input.db, - userId: input.userId, - resource: input.resource, - now, - }), - upgradeHint: buildEntitlementUpgradeHint(input.resource), + updatedAt, }) + if (result.outcome === 'needs_bootstrap') { + throw new Error( + 'UserMeter consume still needs bootstrap after initialize.', + ) + } } - // The fresh-row INSERT branch is unconditional, so a limit below one - // unit can never be satisfied and must be rejected up front. - if (limit < 1) { - await throwLimitError() - } - const result = await input.db - .prepare( - `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) - VALUES (?, ?, ?, 1, ?) - ON CONFLICT(user_id, resource, day) DO UPDATE SET - count = entitlement_daily_counters.count + 1, - updated_at = excluded.updated_at - WHERE entitlement_daily_counters.count + 1 <= ?`, - ) - .bind( - input.userId, - input.resource, - utcDayKey(now), - now.toISOString(), + if (!result.consumed) { + throw new EntitlementLimitError({ + resource, + plan, limit, - ) - .run() - if ((result.meta.changes ?? 0) === 0) { - await throwLimitError() + current: result.count, + upgradeHint: buildEntitlementUpgradeHint(resource), + }) } + scheduleDailyEntitlementMirror( + mirrorDailyEntitlementAbsoluteCount({ + db: input.db, + userId: input.userId, + resource, + day, + count: result.count, + mirrorUpdatedAt: result.mirrorUpdatedAt, + }), + input.waitUntil, + ) } -/** - * Atomically refund one previously consumed daily entitlement unit for the - * given user/resource/UTC day (floors at zero). Scoped by `user_id` so a - * refund can never touch another user's counter. Used when inbound receive - * quota was charged before a retryable storage failure — the same `now` - * (day key) as the matching `consumeDailyEntitlement` call must be passed. - */ -export async function refundDailyEntitlement(input: { +export type RefundDailyEntitlementInput = { db: D1Database + env: UserMeterEnv userId: string resource: EntitlementResource now?: Date -}): Promise { + waitUntil?: (promise: Promise) => void +} + +/** + * Atomically refund one previously consumed daily entitlement unit in + * UserMeter (floors at zero). Pass the same `now` (day key) as the matching + * consume. The legacy D1 mirror is best-effort and never awaited. + */ +export async function refundDailyEntitlement( + input: RefundDailyEntitlementInput, +): Promise { + const resource = assertDailyEntitlementResource(input.resource) const now = input.now ?? new Date() - await input.db - .prepare( - `UPDATE entitlement_daily_counters - SET count = MAX(0, count - 1), - updated_at = ? - WHERE user_id = ? - AND resource = ? - AND day = ?`, - ) - .bind(now.toISOString(), input.userId, input.resource, utcDayKey(now)) - .run() + const day = utcDayKey(now) + const updatedAt = now.toISOString() + const meter = userMeterRpc({ env: input.env, userId: input.userId }) + const result = await meter.refund({ + resource, + day, + updatedAt, + }) + // revision 0 means the key was never initialized; nothing to mirror. + if (result.revision < 1) return + scheduleDailyEntitlementMirror( + mirrorDailyEntitlementAbsoluteCount({ + db: input.db, + userId: input.userId, + resource, + day, + count: result.count, + mirrorUpdatedAt: result.mirrorUpdatedAt, + }), + input.waitUntil, + ) } diff --git a/packages/worker/src/entitlements/user-meter-client.ts b/packages/worker/src/entitlements/user-meter-client.ts new file mode 100644 index 0000000000..e8e27a2019 --- /dev/null +++ b/packages/worker/src/entitlements/user-meter-client.ts @@ -0,0 +1,28 @@ +import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { type UserMeterRpc } from './user-meter-do.ts' + +export type UserMeterEnv = { + USER_METER?: DurableObjectNamespace +} + +export function userMeterNamespace( + env: UserMeterEnv, +): DurableObjectNamespace | null { + return env.USER_METER ?? null +} + +/** Typed per-user UserMeter RPC stub; throws when `USER_METER` is missing. */ +export function userMeterRpc(input: { + env: UserMeterEnv + userId: string +}): UserMeterRpc { + const namespace = userMeterNamespace(input.env) + if (!namespace) { + throw new Error('USER_METER Durable Object binding is not configured.') + } + return namespace.get( + namespace.idFromName(userMeterDurableObjectName(input.userId)), + ) as unknown as UserMeterRpc +} + +export type { UserMeterRpc } diff --git a/packages/worker/src/entitlements/user-meter-do.ts b/packages/worker/src/entitlements/user-meter-do.ts new file mode 100644 index 0000000000..36ac9c5213 --- /dev/null +++ b/packages/worker/src/entitlements/user-meter-do.ts @@ -0,0 +1,678 @@ +import * as Sentry from '@sentry/cloudflare' +import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { DurableObject } from 'cloudflare:workers' +import { buildSentryOptions } from '#worker/sentry-options.ts' +import { type EntitlementResource } from './plans.ts' + +/** Daily rate-style resources stored in the per-user UserMeter (UTC day keys). */ +export const dailyEntitlementResources = [ + 'email_sends_per_day', + 'email_receives_per_day', + 'execute_calls_per_day', + 'outbound_fetches_per_day', +] as const satisfies ReadonlyArray + +export type DailyEntitlementResource = + (typeof dailyEntitlementResources)[number] + +export function isDailyEntitlementResource( + resource: string, +): resource is DailyEntitlementResource { + return (dailyEntitlementResources as ReadonlyArray).includes(resource) +} + +/** Retention window for UserMeter daily rows (enforcement needs today only). */ +export const userMeterDailyCounterRetentionDays = 7 + +const metaSchemaVersionKey = 'schema_version' +/** Bump when initializeSchema DDL changes; warm objects skip DDL. */ +const userMeterSchemaVersion = 3 + +const defaultExportPageSize = 100 +const maxExportPageSize = 500 +const maxInboundDeliveryIdLength = 256 +const inboundReceiveResource = + 'email_receives_per_day' satisfies DailyEntitlementResource +const utcDayKeyPattern = /^\d{4}-\d{2}-\d{2}$/ + +/** + * Legacy D1 mirror `updated_at` token. Lexicographic order matches revision + * order; `r/` sorts after pre-cutover ISO timestamps. + */ +export function userMeterMirrorUpdatedAtToken(revision: number): string { + const safeRevision = + Number.isSafeInteger(revision) && revision > 0 ? revision : 0 + return `r/${String(safeRevision).padStart(20, '0')}` +} + +export type UserMeterCounterRow = { + resource: DailyEntitlementResource + day: string + count: number + revision: number + updatedAt: string + mirrorUpdatedAt: string +} + +export type UserMeterReadyState = { + outcome: 'ready' + count: number + revision: number + mirrorUpdatedAt: string +} + +export type UserMeterBootstrapState = { + outcome: 'needs_bootstrap' +} + +export type UserMeterConsumeResult = + | UserMeterBootstrapState + | (UserMeterReadyState & { consumed: boolean }) + +export type UserMeterReadResult = UserMeterBootstrapState | UserMeterReadyState + +export type UserMeterRefundResult = UserMeterReadyState + +export type UserMeterInitializeResult = UserMeterReadyState & { + created: boolean +} + +export type UserMeterExportResult = { + counters: Array + nextStartAfter: string | null + truncated: boolean +} + +/** + * Inbound delivery claim + receive consume. Retries set `replayed` without + * incrementing; `day`/`resource` come from the original claim on cross-day + * retries. + */ +export type UserMeterInboundDeliveryConsumeResult = + | UserMeterBootstrapState + | (UserMeterReadyState & { + consumed: boolean + replayed: boolean + day: string + resource: typeof inboundReceiveResource + }) + +type ExportCursor = { + day: string + resource: string +} + +function assertDailyResource(resource: string): DailyEntitlementResource { + if (!isDailyEntitlementResource(resource)) { + throw new Error( + `UserMeter resource must be a daily entitlement resource; got ${JSON.stringify(resource)}.`, + ) + } + return resource +} + +function assertInboundReceiveResource( + resource: string, +): typeof inboundReceiveResource { + const daily = assertDailyResource(resource) + if (daily !== inboundReceiveResource) { + throw new Error( + `UserMeter inbound delivery consume requires ${inboundReceiveResource}; got ${JSON.stringify(resource)}.`, + ) + } + return daily +} + +function assertUtcDayKey(day: string): string { + if (!utcDayKeyPattern.test(day)) { + throw new Error( + `UserMeter day must be a UTC YYYY-MM-DD key; got ${JSON.stringify(day)}.`, + ) + } + return day +} + +function assertInboundDeliveryId(deliveryId: string): string { + if ( + typeof deliveryId !== 'string' || + deliveryId.length === 0 || + deliveryId.length > maxInboundDeliveryIdLength + ) { + throw new Error( + `UserMeter inbound deliveryId must be a non-empty string up to ${maxInboundDeliveryIdLength} characters.`, + ) + } + return deliveryId +} + +function retentionCutoffDay(now: Date): string { + const cutoff = new Date(now) + cutoff.setUTCDate( + cutoff.getUTCDate() - (userMeterDailyCounterRetentionDays - 1), + ) + return utcDayKey(cutoff) +} + +function encodeExportCursor(cursor: ExportCursor) { + return JSON.stringify([cursor.day, cursor.resource]) +} + +function decodeExportCursor(startAfter: string): ExportCursor | null { + try { + const parsed = JSON.parse(startAfter) as unknown + if (!Array.isArray(parsed) || parsed.length !== 2) return null + const [day, resource] = parsed + if (typeof day !== 'string' || typeof resource !== 'string') return null + return { day, resource } + } catch { + return null + } +} + +function normalizePageSize(pageSize: number | undefined) { + const requested = + typeof pageSize === 'number' && Number.isFinite(pageSize) + ? Math.trunc(pageSize) + : defaultExportPageSize + return Math.min(Math.max(requested, 1), maxExportPageSize) +} + +function readyState(count: number, revision: number): UserMeterReadyState { + const safeCount = Math.max(0, count) + const safeRevision = Math.max(0, revision) + return { + outcome: 'ready', + count: safeCount, + revision: safeRevision, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(safeRevision), + } +} + +class UserMeterBase extends DurableObject { + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env) + this.ctx.blockConcurrencyWhile(async () => { + this.initializeSchema() + }) + } + + private initializeSchema() { + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS user_meter_meta ( + key TEXT PRIMARY KEY NOT NULL, + value INTEGER NOT NULL + ) + `) + const versionRow = this.ctx.storage.sql + .exec<{ + value: number + }>( + `SELECT value FROM user_meter_meta WHERE key = ? LIMIT 1`, + metaSchemaVersionKey, + ) + .toArray()[0] + const version = versionRow == null ? null : Number(versionRow.value) || 0 + if (version === userMeterSchemaVersion) return + + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS daily_counters ( + resource TEXT NOT NULL, + day TEXT NOT NULL, + count INTEGER NOT NULL, + revision INTEGER NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (resource, day) + ) + `) + if (version === 1) { + try { + this.ctx.storage.sql.exec( + `ALTER TABLE daily_counters ADD COLUMN revision INTEGER NOT NULL DEFAULT 0`, + ) + } catch { + // Column already present on a partially migrated object. + } + } + this.ctx.storage.sql.exec( + `CREATE INDEX IF NOT EXISTS idx_daily_counters_day + ON daily_counters (day)`, + ) + // Per-user delivery-id ledger (PK is delivery_id alone); same retention + // window as daily counters so Email Routing retries cannot double-charge. + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS inbound_delivery_claims ( + delivery_id TEXT PRIMARY KEY NOT NULL, + resource TEXT NOT NULL, + day TEXT NOT NULL, + count_after INTEGER NOT NULL, + revision INTEGER NOT NULL, + claimed_at TEXT NOT NULL + ) + `) + this.ctx.storage.sql.exec( + `CREATE INDEX IF NOT EXISTS idx_inbound_delivery_claims_day + ON inbound_delivery_claims (day)`, + ) + this.ctx.storage.sql.exec( + `INSERT INTO user_meter_meta (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + metaSchemaVersionKey, + userMeterSchemaVersion, + ) + } + + private deleteStaleCounters(now: Date) { + const cutoffDay = retentionCutoffDay(now) + this.ctx.storage.sql.exec( + `DELETE FROM daily_counters WHERE day < ?`, + cutoffDay, + ) + this.ctx.storage.sql.exec( + `DELETE FROM inbound_delivery_claims WHERE day < ?`, + cutoffDay, + ) + } + + private readInboundDeliveryClaim(deliveryId: string): { + resource: string + day: string + countAfter: number + revision: number + claimedAt: string + } | null { + const row = this.ctx.storage.sql + .exec<{ + resource: string + day: string + count_after: number + revision: number + claimed_at: string + }>( + `SELECT resource, day, count_after, revision, claimed_at + FROM inbound_delivery_claims + WHERE delivery_id = ?`, + deliveryId, + ) + .toArray()[0] + if (!row) return null + return { + resource: String(row.resource), + day: String(row.day), + countAfter: Math.max(0, Number(row.count_after ?? 0)), + revision: Math.max(0, Number(row.revision ?? 0)), + claimedAt: String(row.claimed_at ?? ''), + } + } + + private readRow( + resource: DailyEntitlementResource, + day: string, + ): { count: number; revision: number; updatedAt: string } | null { + const row = this.ctx.storage.sql + .exec<{ count: number; revision: number; updated_at: string }>( + `SELECT count, revision, updated_at + FROM daily_counters + WHERE resource = ? AND day = ?`, + resource, + day, + ) + .toArray()[0] + if (!row) return null + return { + count: Math.max(0, Number(row.count ?? 0)), + revision: Math.max(0, Number(row.revision ?? 0)), + updatedAt: String(row.updated_at ?? ''), + } + } + + /** Cold-key seed from legacy D1; INSERT OR IGNORE is concurrency-safe. */ + async initialize(input: { + resource: string + day: string + count: number + updatedAt: string + }): Promise { + const resource = assertDailyResource(input.resource) + const day = assertUtcDayKey(input.day) + const now = new Date(input.updatedAt) + this.deleteStaleCounters(Number.isNaN(now.valueOf()) ? new Date() : now) + const count = Math.max(0, Math.trunc(Number(input.count) || 0)) + const cursor = this.ctx.storage.sql.exec( + `INSERT INTO daily_counters (resource, day, count, revision, updated_at) + VALUES (?, ?, ?, 1, ?) + ON CONFLICT(resource, day) DO NOTHING`, + resource, + day, + count, + input.updatedAt, + ) + const created = cursor.rowsWritten > 0 + const row = this.readRow(resource, day) + if (!row) { + throw new Error( + 'UserMeter initialize failed to materialize a counter row.', + ) + } + return { ...readyState(row.count, row.revision), created } + } + + /** Check-and-increment one unit; missing keys return `needs_bootstrap`. */ + async consume(input: { + resource: string + day: string + limit: number + updatedAt: string + }): Promise { + const resource = assertDailyResource(input.resource) + const day = assertUtcDayKey(input.day) + const now = new Date(input.updatedAt) + this.deleteStaleCounters(Number.isNaN(now.valueOf()) ? new Date() : now) + + const existing = this.readRow(resource, day) + if (!existing) { + return { outcome: 'needs_bootstrap' } + } + + if (input.limit < 1 || existing.count + 1 > input.limit) { + return { + ...readyState(existing.count, existing.revision), + consumed: false, + } + } + + const nextCount = existing.count + 1 + const nextRevision = existing.revision + 1 + this.ctx.storage.sql.exec( + `UPDATE daily_counters + SET count = ?, + revision = ?, + updated_at = ? + WHERE resource = ? AND day = ? AND revision = ?`, + nextCount, + nextRevision, + input.updatedAt, + resource, + day, + existing.revision, + ) + const row = this.readRow(resource, day) + if (!row) { + throw new Error('UserMeter consume lost the counter row.') + } + return { + ...readyState(row.count, row.revision), + consumed: row.count === nextCount && row.revision === nextRevision, + } + } + + async read(input: { + resource: string + day: string + now?: string + }): Promise { + const resource = assertDailyResource(input.resource) + const day = assertUtcDayKey(input.day) + const now = input.now ? new Date(input.now) : new Date() + this.deleteStaleCounters(Number.isNaN(now.valueOf()) ? new Date() : now) + const row = this.readRow(resource, day) + if (!row) return { outcome: 'needs_bootstrap' } + return readyState(row.count, row.revision) + } + + /** + * Claim `deliveryId` and consume one receive unit. Increment + claim insert + * run in `storage.transactionSync`. + */ + async consumeInboundDelivery(input: { + deliveryId: string + resource: string + day: string + limit: number + updatedAt: string + }): Promise { + const resource = assertInboundReceiveResource(input.resource) + const day = assertUtcDayKey(input.day) + const deliveryId = assertInboundDeliveryId(input.deliveryId) + const now = new Date(input.updatedAt) + this.deleteStaleCounters(Number.isNaN(now.valueOf()) ? new Date() : now) + + const priorClaim = this.readInboundDeliveryClaim(deliveryId) + if (priorClaim) { + if (priorClaim.resource !== resource) { + throw new Error( + `UserMeter inbound deliveryId was claimed for ${JSON.stringify(priorClaim.resource)}; cannot reuse for ${JSON.stringify(resource)}.`, + ) + } + const claimedResource = assertInboundReceiveResource(priorClaim.resource) + const claimedDay = assertUtcDayKey(priorClaim.day) + const row = this.readRow(claimedResource, claimedDay) + return { + ...(row + ? readyState(row.count, row.revision) + : readyState(priorClaim.countAfter, priorClaim.revision)), + consumed: false, + replayed: true, + day: claimedDay, + resource: claimedResource, + } + } + + const existing = this.readRow(resource, day) + if (!existing) { + return { outcome: 'needs_bootstrap' } + } + + if (input.limit < 1 || existing.count + 1 > input.limit) { + return { + ...readyState(existing.count, existing.revision), + consumed: false, + replayed: false, + day, + resource, + } + } + + const nextCount = existing.count + 1 + const nextRevision = existing.revision + 1 + this.ctx.storage.transactionSync(() => { + this.ctx.storage.sql.exec( + `UPDATE daily_counters + SET count = ?, + revision = ?, + updated_at = ? + WHERE resource = ? AND day = ? AND revision = ?`, + nextCount, + nextRevision, + input.updatedAt, + resource, + day, + existing.revision, + ) + this.ctx.storage.sql.exec( + `INSERT INTO inbound_delivery_claims ( + delivery_id, resource, day, count_after, revision, claimed_at + ) VALUES (?, ?, ?, ?, ?, ?)`, + deliveryId, + resource, + day, + nextCount, + nextRevision, + input.updatedAt, + ) + }) + const row = this.readRow(resource, day) + if (!row) { + throw new Error( + 'UserMeter inbound delivery consume lost the counter row.', + ) + } + return { + ...readyState(row.count, row.revision), + consumed: row.count === nextCount && row.revision === nextRevision, + replayed: false, + day, + resource, + } + } + + /** Decrement one unit (floors at zero); missing keys stay uninitialized. */ + async refund(input: { + resource: string + day: string + updatedAt: string + }): Promise { + const resource = assertDailyResource(input.resource) + const day = assertUtcDayKey(input.day) + const now = new Date(input.updatedAt) + this.deleteStaleCounters(Number.isNaN(now.valueOf()) ? new Date() : now) + + const existing = this.readRow(resource, day) + if (!existing) { + return readyState(0, 0) + } + const nextCount = Math.max(0, existing.count - 1) + const nextRevision = existing.revision + 1 + this.ctx.storage.sql.exec( + `UPDATE daily_counters + SET count = ?, + revision = ?, + updated_at = ? + WHERE resource = ? AND day = ? AND revision = ?`, + nextCount, + nextRevision, + input.updatedAt, + resource, + day, + existing.revision, + ) + const row = this.readRow(resource, day) + if (!row) { + throw new Error('UserMeter refund lost the counter row.') + } + return readyState(row.count, row.revision) + } + + async purge(): Promise<{ ok: true }> { + await this.ctx.storage.deleteAll() + this.initializeSchema() + return { ok: true } + } + + async exportCounters(input: { + pageSize?: number + startAfter?: string | null + }): Promise { + this.deleteStaleCounters(new Date()) + const pageSize = normalizePageSize(input.pageSize) + const cursor = + typeof input.startAfter === 'string' && input.startAfter.length > 0 + ? decodeExportCursor(input.startAfter) + : null + + const rows = ( + cursor + ? this.ctx.storage.sql.exec<{ + resource: string + day: string + count: number + revision: number + updated_at: string + }>( + `SELECT resource, day, count, revision, updated_at + FROM daily_counters + WHERE day > ? + OR (day = ? AND resource > ?) + ORDER BY day ASC, resource ASC + LIMIT ?`, + cursor.day, + cursor.day, + cursor.resource, + pageSize + 1, + ) + : this.ctx.storage.sql.exec<{ + resource: string + day: string + count: number + revision: number + updated_at: string + }>( + `SELECT resource, day, count, revision, updated_at + FROM daily_counters + ORDER BY day ASC, resource ASC + LIMIT ?`, + pageSize + 1, + ) + ).toArray() + + const truncated = rows.length > pageSize + const pageRows = truncated ? rows.slice(0, pageSize) : rows + const counters: Array = [] + for (const row of pageRows) { + const resource = String(row.resource) + if (!isDailyEntitlementResource(resource)) continue + const revision = Math.max(0, Number(row.revision ?? 0)) + counters.push({ + resource, + day: String(row.day), + count: Math.max(0, Number(row.count ?? 0)), + revision, + updatedAt: String(row.updated_at), + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(revision), + }) + } + const last = pageRows[pageRows.length - 1] + return { + counters, + nextStartAfter: + truncated && last + ? encodeExportCursor({ + day: String(last.day), + resource: String(last.resource), + }) + : null, + truncated, + } + } +} + +export const UserMeter = Sentry.instrumentDurableObjectWithSentry( + (env: Env) => buildSentryOptions(env), + UserMeterBase, +) + +export type UserMeterRpc = { + initialize: (input: { + resource: string + day: string + count: number + updatedAt: string + }) => Promise + consume: (input: { + resource: string + day: string + limit: number + updatedAt: string + }) => Promise + consumeInboundDelivery: (input: { + deliveryId: string + resource: string + day: string + limit: number + updatedAt: string + }) => Promise + read: (input: { + resource: string + day: string + now?: string + }) => Promise + refund: (input: { + resource: string + day: string + updatedAt: string + }) => Promise + purge: () => Promise<{ ok: true }> + exportCounters: (input: { + pageSize?: number + startAfter?: string | null + }) => Promise +} diff --git a/packages/worker/src/entitlements/user-meter.workers.test.ts b/packages/worker/src/entitlements/user-meter.workers.test.ts new file mode 100644 index 0000000000..4bfec33bfc --- /dev/null +++ b/packages/worker/src/entitlements/user-meter.workers.test.ts @@ -0,0 +1,616 @@ +import { runInDurableObject } from 'cloudflare:test' +import { env } from 'cloudflare:workers' +import { expect, test } from 'vitest' +import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { seedAccount } from '#worker/test-support/workers-seed.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { EntitlementLimitError } from './errors.ts' +import { planLimits } from './plans.ts' +import { consumeDailyEntitlement, refundDailyEntitlement } from './service.ts' +import { ensureEntitlementTestSchema } from './test-schema.ts' +import { userMeterRpc } from './user-meter-client.ts' +import { UserMeter, userMeterMirrorUpdatedAtToken } from './user-meter-do.ts' +import { + createWaitUntilDrain, + withPatchedDbPrepare, +} from '#worker/test-support/user-meter.ts' + +async function seedFreeUser(emailPrefix: string) { + await ensureEntitlementTestSchema(env.APP_DB) + const email = `${emailPrefix}-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(email) + await seedAccount({ + db: env.APP_DB, + email, + username: `meter-${crypto.randomUUID().slice(0, 8)}`, + plan: 'free', + stableUserId: userId, + }) + return { email, userId } +} + +async function readD1DailyCount(input: { + userId: string + resource: string + day: string +}) { + const row = await env.APP_DB.prepare( + `SELECT count, updated_at FROM entitlement_daily_counters + WHERE user_id = ? AND resource = ? AND day = ?`, + ) + .bind(input.userId, input.resource, input.day) + .first<{ count: number; updated_at: string }>() + return row + ? { count: Number(row.count), updatedAt: String(row.updated_at) } + : null +} + +async function waitFor( + predicate: () => boolean, + timeoutMs = 5_000, + label = 'condition', +) { + const started = Date.now() + while (!predicate()) { + if (Date.now() - started > timeoutMs) { + throw new Error(`Timed out waiting for ${label}.`) + } + await new Promise((resolve) => setTimeout(resolve, 5)) + } +} + +const mirrorSql = `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(user_id, resource, day) DO UPDATE SET + count = excluded.count, + updated_at = excluded.updated_at + WHERE entitlement_daily_counters.updated_at < excluded.updated_at` + +test('UserMeter bootstraps from legacy D1 once, then warms without APP_DB awaits', async () => { + const now = new Date('2026-07-31T15:00:00.000Z') + const day = utcDayKey(now) + const sendLimit = planLimits.free.maxEmailSendsPerDay + const user = await seedFreeUser('meter-bootstrap') + const drain = createWaitUntilDrain() + const meter = userMeterRpc({ env, userId: user.userId }) + + const legacyCount = sendLimit - 1 + await env.APP_DB.prepare( + `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) + VALUES (?, ?, ?, ?, ?)`, + ) + .bind( + user.userId, + 'email_sends_per_day', + day, + legacyCount, + '2026-07-31T12:00:00.000Z', + ) + .run() + + let d1PointReads = 0 + using _patch = withPatchedDbPrepare(env.APP_DB, (originalPrepare) => { + return ((query: string) => { + const statement = originalPrepare(query) + const isPointRead = + query.includes('SELECT count FROM entitlement_daily_counters') && + query.includes('WHERE user_id = ? AND resource = ? AND day = ?') + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: async () => { + if (isPointRead) d1PointReads += 1 + return await bound.first() + }, + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: bound.run.bind(bound), + } + }, + } + }) as D1Database['prepare'] + }) + + await consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + expect(d1PointReads).toBe(1) + expect( + await meter.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ + outcome: 'ready', + count: sendLimit, + }) + + const denied = await consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(denied).toBeInstanceOf(EntitlementLimitError) + expect(denied).toMatchObject({ + details: { + resource: 'email_sends_per_day', + plan: 'free', + limit: sendLimit, + current: sendLimit, + }, + }) + expect(d1PointReads).toBe(1) + + await refundDailyEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + await drain.drain() + d1PointReads = 0 + let nonMirrorAppDbRuns = 0 + let releaseMirror!: () => void + const mirrorGate = new Promise((resolve) => { + releaseMirror = resolve + }) + let mirrorRunStarted = false + using _warmPatch = withPatchedDbPrepare(env.APP_DB, (originalPrepare) => { + return ((query: string) => { + const statement = originalPrepare(query) + const isMirrorWrite = + query.includes('INSERT INTO entitlement_daily_counters') && + query.includes('updated_at < excluded.updated_at') + const isPointRead = + query.includes('SELECT count FROM entitlement_daily_counters') && + query.includes('WHERE user_id = ? AND resource = ? AND day = ?') + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: async () => { + if (isPointRead) d1PointReads += 1 + return await bound.first() + }, + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: async () => { + if (isMirrorWrite) { + mirrorRunStarted = true + await mirrorGate + return await bound.run() + } + nonMirrorAppDbRuns += 1 + return await bound.run() + }, + } + }, + } + }) as D1Database['prepare'] + }) + try { + await expect( + consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }), + ).resolves.toBeUndefined() + expect(mirrorRunStarted).toBe(true) + expect(d1PointReads).toBe(0) + expect(nonMirrorAppDbRuns).toBe(0) + } finally { + releaseMirror() + } + await drain.drain() +}, 30_000) + +test('UserMeter mirror tokens reject out-of-order absolute D1 writes', async () => { + const now = new Date('2026-07-31T16:00:00.000Z') + const day = utcDayKey(now) + const user = await seedFreeUser('meter-mirror-order') + const meter = userMeterRpc({ env, userId: user.userId }) + + await meter.initialize({ + resource: 'email_sends_per_day', + day, + count: 0, + updatedAt: now.toISOString(), + }) + const first = await meter.consume({ + resource: 'email_sends_per_day', + day, + limit: 100, + updatedAt: now.toISOString(), + }) + expect(first).toMatchObject({ + outcome: 'ready', + consumed: true, + count: 1, + revision: 2, + }) + const second = await meter.consume({ + resource: 'email_sends_per_day', + day, + limit: 100, + updatedAt: now.toISOString(), + }) + expect(second).toMatchObject({ + outcome: 'ready', + consumed: true, + count: 2, + revision: 3, + }) + const refunded = await meter.refund({ + resource: 'email_sends_per_day', + day, + updatedAt: now.toISOString(), + }) + expect(refunded).toMatchObject({ + outcome: 'ready', + count: 1, + revision: 4, + }) + if (first.outcome !== 'ready') { + throw new Error('Expected ready consume results.') + } + + type PendingMirror = { + count: number + mirrorUpdatedAt: string + resolve: () => void + promise: Promise + } + const pending: Array = [] + function deferMirror(count: number, mirrorUpdatedAt: string) { + let resolve!: () => void + const promise = new Promise((res) => { + resolve = res + }) + pending.push({ count, mirrorUpdatedAt, resolve, promise }) + return promise + } + + using _patch = withPatchedDbPrepare(env.APP_DB, (originalPrepare) => { + return ((query: string) => { + const statement = originalPrepare(query) + const isMirrorWrite = + query.includes('INSERT INTO entitlement_daily_counters') && + query.includes('updated_at < excluded.updated_at') + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: bound.first.bind(bound), + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: async () => { + if (!isMirrorWrite) return await bound.run() + await deferMirror(Number(params[3]), String(params[4])) + return await bound.run() + }, + } + }, + } + }) as D1Database['prepare'] + }) + + const older = env.APP_DB.prepare(mirrorSql) + .bind( + user.userId, + 'email_sends_per_day', + day, + first.count, + first.mirrorUpdatedAt, + ) + .run() + const newer = env.APP_DB.prepare(mirrorSql) + .bind( + user.userId, + 'email_sends_per_day', + day, + refunded.count, + refunded.mirrorUpdatedAt, + ) + .run() + + await waitFor(() => pending.length === 2, 5_000, 'mirror gates') + expect(pending.map((entry) => entry.mirrorUpdatedAt)).toEqual([ + userMeterMirrorUpdatedAtToken(2), + userMeterMirrorUpdatedAtToken(4), + ]) + const [olderGate, newerGate] = pending + newerGate!.resolve() + await newer + expect( + await readD1DailyCount({ + userId: user.userId, + resource: 'email_sends_per_day', + day, + }), + ).toEqual({ + count: 1, + updatedAt: userMeterMirrorUpdatedAtToken(4), + }) + + olderGate!.resolve() + await older + expect( + await readD1DailyCount({ + userId: user.userId, + resource: 'email_sends_per_day', + day, + }), + ).toEqual({ + count: 1, + updatedAt: userMeterMirrorUpdatedAtToken(4), + }) + expect( + await meter.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 1, revision: 4 }) +}, 30_000) + +test('UserMeter daily entitlement consume/refund/read/export/purge workflow is per-user and mirror-safe', async () => { + const now = new Date('2026-07-31T15:00:00.000Z') + const day = utcDayKey(now) + const sendLimit = planLimits.free.maxEmailSendsPerDay + const userA = await seedFreeUser('meter-a') + const userB = await seedFreeUser('meter-b') + const drain = createWaitUntilDrain() + const meterA = userMeterRpc({ env, userId: userA.userId }) + const meterB = userMeterRpc({ env, userId: userB.userId }) + + expect(userMeterDurableObjectName(userA.userId)).toBe(userA.userId) + + await meterA.initialize({ + resource: 'email_sends_per_day', + day, + count: 0, + updatedAt: now.toISOString(), + }) + await expect( + meterA.consume({ + resource: 'email_sends_per_day', + day, + limit: sendLimit, + updatedAt: now.toISOString(), + }), + ).resolves.toMatchObject({ + outcome: 'ready', + consumed: true, + count: 1, + }) + + await consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: userA.userId, + email: userA.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + await drain.drain() + expect( + await meterA.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 2 }) + + for (let index = 2; index < sendLimit; index += 1) { + await consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: userA.userId, + email: userA.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + } + const concurrent = await Promise.all( + Array.from({ length: 8 }, async () => + consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: userA.userId, + email: userA.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }).then( + () => null, + (thrown: unknown) => thrown, + ), + ), + ) + expect(concurrent.filter((result) => result === null)).toHaveLength(0) + const denials = concurrent.filter( + (result) => result instanceof EntitlementLimitError, + ) + expect(denials).toHaveLength(8) + for (const denial of denials) { + expect(denial).toMatchObject({ + details: { + code: 'entitlement_limit_exceeded', + resource: 'email_sends_per_day', + plan: 'free', + limit: sendLimit, + current: sendLimit, + }, + }) + } + + await consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: userB.userId, + email: userB.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + expect( + await meterB.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 1 }) + expect( + await meterA.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: sendLimit }) + + await refundDailyEntitlement({ + db: env.APP_DB, + env, + userId: userA.userId, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + expect( + await meterA.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: sendLimit - 1 }) + for (let index = 0; index < sendLimit; index += 1) { + await refundDailyEntitlement({ + db: env.APP_DB, + env, + userId: userA.userId, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + } + expect( + await meterA.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 0 }) + + await consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: userA.userId, + email: userA.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }) + await consumeDailyEntitlement({ + db: env.APP_DB, + env, + userId: userA.userId, + email: userA.email, + resource: 'execute_calls_per_day', + now, + waitUntil: drain.waitUntil, + }) + const exportedA = await meterA.exportCounters({}) + expect(exportedA.counters).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + resource: 'email_sends_per_day', + day, + count: 1, + }), + expect.objectContaining({ + resource: 'execute_calls_per_day', + day, + count: 1, + }), + ]), + ) + + await meterA.purge() + expect(await meterA.read({ resource: 'email_sends_per_day', day })).toEqual({ + outcome: 'needs_bootstrap', + }) + expect(await meterA.exportCounters({})).toEqual({ + counters: [], + nextStartAfter: null, + truncated: false, + }) + expect( + await meterB.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 1 }) + + await env.APP_DB.prepare( + `DELETE FROM entitlement_daily_counters + WHERE user_id = ? AND resource = ? AND day = ?`, + ) + .bind(userA.userId, 'email_sends_per_day', day) + .run() + + consoleWarn.mockImplementation(() => {}) + const failingMirrorDb = { + prepare(query: string) { + if ( + query.includes('INSERT INTO entitlement_daily_counters') && + query.includes('updated_at < excluded.updated_at') + ) { + return { + bind() { + return { + async run() { + throw new Error('mirror write failed') + }, + } + }, + } + } + return env.APP_DB.prepare(query) + }, + } as unknown as D1Database + + await expect( + consumeDailyEntitlement({ + db: failingMirrorDb, + env, + userId: userA.userId, + email: userA.email, + resource: 'email_sends_per_day', + now, + waitUntil: drain.waitUntil, + }), + ).resolves.toBeUndefined() + await drain.drain() + expect( + await meterA.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 1 }) + expect(consoleWarn).toHaveBeenCalledWith( + 'entitlement-daily-mirror-failed', + expect.any(Error), + ) +}, 30_000) + +test('UserMeter consume rejects invalid UTC day keys before SQL', async () => { + const user = await seedFreeUser('meter-invalid-day') + const stub = env.USER_METER.get( + env.USER_METER.idFromName(userMeterDurableObjectName(user.userId)), + ) + await runInDurableObject(stub, async (instance: UserMeter) => { + await expect( + instance.consume({ + resource: 'email_sends_per_day', + day: 'not-a-day', + limit: 1, + updatedAt: '2026-07-31T15:00:00.000Z', + }), + ).rejects.toThrow(/UTC YYYY-MM-DD/) + }) +}, 30_000) diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 33f7fe0be5..0d50120601 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -6,6 +6,7 @@ import { MCP } from './mcp/index.ts' import { JobManager } from './jobs/manager-do.ts' import { StorageRunner } from './storage-runner.ts' import { RunLog } from './run-records/run-log-do.ts' +import { UserMeter } from './entitlements/user-meter-do.ts' import { RepoSession } from './repo/repo-session-do.ts' import { PackageRealtimeSession } from '#worker/package-runtime/realtime-session.ts' import { PackageServiceInstance } from '#worker/package-runtime/package-service.ts' @@ -108,6 +109,7 @@ export { PackageAppRuntimeBridge, StorageRunner, RunLog, + UserMeter, OAuthPurgeCoordinator, } diff --git a/packages/worker/src/mcp/capabilities/email/email-usage-get.ts b/packages/worker/src/mcp/capabilities/email/email-usage-get.ts index 5e3e6436ab..e1db22b96d 100644 --- a/packages/worker/src/mcp/capabilities/email/email-usage-get.ts +++ b/packages/worker/src/mcp/capabilities/email/email-usage-get.ts @@ -8,6 +8,7 @@ import { } from '#worker/entitlements/plans.ts' import { getUserPlan, + readDailyEntitlementResourceUsage, readEntitlementResourceUsage, } from '#worker/entitlements/service.ts' import { utcDayKey } from '@kody-internal/shared/date-keys.ts' @@ -54,14 +55,16 @@ export const emailUsageGetCapability = defineDomainCapability( resource: 'stored_email_messages', now, }), - readEntitlementResourceUsage({ + readDailyEntitlementResourceUsage({ db, + env: ctx.env, userId: user.userId, resource: 'email_sends_per_day', now, }), - readEntitlementResourceUsage({ + readDailyEntitlementResourceUsage({ db, + env: ctx.env, userId: user.userId, resource: 'email_receives_per_day', now, diff --git a/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts b/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts index 2a59e9abb5..2b71fd5734 100644 --- a/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts @@ -2,6 +2,7 @@ import { env } from 'cloudflare:workers' import { expect, test } from 'vitest' import { createMcpCallerContext } from '#mcp/context.ts' import { maxPlanEmailLimits, planLimits } from '#worker/entitlements/plans.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' import { utcDayKey } from '@kody-internal/shared/date-keys.ts' import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' import { seedAccount } from '#worker/test-support/workers-seed.ts' @@ -62,103 +63,157 @@ function buildCallerContext(user: { userId: string; email: string } | null) { }) } -test('email_usage_get returns usage for verified users and enforces auth requirements', async () => { - await ensureEmailTestSchema(env.APP_DB) - await expect( - emailUsageGetCapability.handler( - {}, - { env, callerContext: buildCallerContext(null) }, - ), - ).rejects.toThrow(/Authenticated MCP user/) +test( + 'email_usage_get enforces auth and reads UserMeter-backed daily counts', + { timeout: 30_000 }, + async () => { + await ensureEmailTestSchema(env.APP_DB) + await expect( + emailUsageGetCapability.handler( + {}, + { env, callerContext: buildCallerContext(null) }, + ), + ).rejects.toThrow(/Authenticated MCP user/) - const unverifiedEmail = `usage-unverified-${crypto.randomUUID()}@example.com` - const unverifiedUserId = await createStableUserIdFromEmail(unverifiedEmail) - await seedUsageAccount({ - email: unverifiedEmail, - plan: 'max', - emailVerifiedAt: null, - }) - await expect( - emailUsageGetCapability.handler( + const unverifiedEmail = `usage-unverified-${crypto.randomUUID()}@example.com` + const unverifiedUserId = await createStableUserIdFromEmail(unverifiedEmail) + await seedUsageAccount({ + email: unverifiedEmail, + plan: 'max', + emailVerifiedAt: null, + }) + await expect( + emailUsageGetCapability.handler( + {}, + { + env, + callerContext: buildCallerContext({ + userId: unverifiedUserId, + email: unverifiedEmail, + }), + }, + ), + ).rejects.toThrow(/Account email is not verified/) + + const planEmail = `usage-plan-${crypto.randomUUID()}@example.com` + const planUserId = await createStableUserIdFromEmail(planEmail) + await seedUsageAccount({ email: planEmail, plan: 'pro' }) + await seedDailyCounter({ + userId: planUserId, + resource: 'email_sends_per_day', + count: 3, + }) + await seedDailyCounter({ + userId: planUserId, + resource: 'email_receives_per_day', + count: 5, + }) + await seedStoredMessages(planUserId, 2) + + const planResult = await emailUsageGetCapability.handler( {}, { env, callerContext: buildCallerContext({ - userId: unverifiedUserId, - email: unverifiedEmail, + userId: planUserId, + email: planEmail, }), }, - ), - ).rejects.toThrow(/Account email is not verified/) + ) + expect(planResult).toEqual({ + plan: 'pro', + day: utcDayKey(), + stored_messages: { + count: 2, + limit: planLimits.pro.maxStoredEmailMessages, + }, + sends_today: { count: 3, limit: planLimits.pro.maxEmailSendsPerDay }, + receives_today: { + count: 5, + limit: planLimits.pro.maxEmailReceivesPerDay, + }, + max_message_bytes: planLimits.pro.maxEmailMessageBytes, + }) + const day = utcDayKey() + const planMeter = userMeterRpc({ env, userId: planUserId }) + expect( + await planMeter.read({ resource: 'email_sends_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 3 }) + expect( + await planMeter.read({ resource: 'email_receives_per_day', day }), + ).toMatchObject({ outcome: 'ready', count: 5 }) - const planEmail = `usage-plan-${crypto.randomUUID()}@example.com` - const planUserId = await createStableUserIdFromEmail(planEmail) - await seedUsageAccount({ email: planEmail, plan: 'pro' }) - await seedDailyCounter({ - userId: planUserId, - resource: 'email_sends_per_day', - count: 3, - }) - await seedDailyCounter({ - userId: planUserId, - resource: 'email_receives_per_day', - count: 5, - }) - await seedStoredMessages(planUserId, 2) + const maxEmail = `usage-max-${crypto.randomUUID()}@example.com` + const maxUserId = await createStableUserIdFromEmail(maxEmail) + await seedUsageAccount({ email: maxEmail, plan: 'max' }) + const maxResult = await emailUsageGetCapability.handler( + {}, + { + env, + callerContext: buildCallerContext({ + userId: maxUserId, + email: maxEmail, + }), + }, + ) + expect(maxResult).toEqual({ + plan: 'max', + day: utcDayKey(), + stored_messages: { + count: 0, + limit: maxPlanEmailLimits.stored_email_messages, + }, + sends_today: { + count: 0, + limit: maxPlanEmailLimits.email_sends_per_day, + }, + receives_today: { + count: 0, + limit: maxPlanEmailLimits.email_receives_per_day, + }, + max_message_bytes: maxPlanEmailLimits.email_message_bytes, + }) - const planResult = await emailUsageGetCapability.handler( - {}, - { - env, - callerContext: buildCallerContext({ - userId: planUserId, - email: planEmail, - }), - }, - ) - expect(planResult).toEqual({ - plan: 'pro', - day: utcDayKey(), - stored_messages: { - count: 2, - limit: planLimits.pro.maxStoredEmailMessages, - }, - sends_today: { count: 3, limit: planLimits.pro.maxEmailSendsPerDay }, - receives_today: { + const meterEmail = `usage-meter-${crypto.randomUUID()}@example.com` + const meterUserId = await createStableUserIdFromEmail(meterEmail) + await seedUsageAccount({ email: meterEmail, plan: 'pro' }) + await seedDailyCounter({ + userId: meterUserId, + resource: 'email_sends_per_day', + count: 3, + }) + await seedDailyCounter({ + userId: meterUserId, + resource: 'email_receives_per_day', count: 5, - limit: planLimits.pro.maxEmailReceivesPerDay, - }, - max_message_bytes: planLimits.pro.maxEmailMessageBytes, - }) - - const maxEmail = `usage-max-${crypto.randomUUID()}@example.com` - const maxUserId = await createStableUserIdFromEmail(maxEmail) - await seedUsageAccount({ email: maxEmail, plan: 'max' }) - const maxResult = await emailUsageGetCapability.handler( - {}, - { - env, - callerContext: buildCallerContext({ - userId: maxUserId, - email: maxEmail, - }), - }, - ) - expect(maxResult).toEqual({ - plan: 'max', - day: utcDayKey(), - stored_messages: { - count: 0, - limit: maxPlanEmailLimits.stored_email_messages, - }, - sends_today: { - count: 0, - limit: maxPlanEmailLimits.email_sends_per_day, - }, - receives_today: { - count: 0, - limit: maxPlanEmailLimits.email_receives_per_day, - }, - max_message_bytes: maxPlanEmailLimits.email_message_bytes, - }) -}) + }) + await seedStoredMessages(meterUserId, 2) + const updatedAt = new Date().toISOString() + const meter = userMeterRpc({ env, userId: meterUserId }) + await meter.initialize({ + resource: 'email_sends_per_day', + day, + count: 13, + updatedAt, + }) + await meter.initialize({ + resource: 'email_receives_per_day', + day, + count: 15, + updatedAt, + }) + const meterResult = await emailUsageGetCapability.handler( + {}, + { + env, + callerContext: buildCallerContext({ + userId: meterUserId, + email: meterEmail, + }), + }, + ) + expect(meterResult.sends_today.count).toBe(13) + expect(meterResult.receives_today.count).toBe(15) + expect(meterResult.stored_messages.count).toBe(2) + }, +) diff --git a/packages/worker/src/mcp/fetch-gateway.node.test.ts b/packages/worker/src/mcp/fetch-gateway.node.test.ts index 66cbdf1b10..189235e61e 100644 --- a/packages/worker/src/mcp/fetch-gateway.node.test.ts +++ b/packages/worker/src/mcp/fetch-gateway.node.test.ts @@ -12,15 +12,10 @@ import { import { buildBasicAuthSecretPlaceholder } from '#mcp/secrets/placeholders.ts' import * as secretService from '#mcp/secrets/service.ts' import * as communityRepo from '#worker/community/repo.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' import * as packageRepo from '#worker/package-registry/repo.ts' -/** - * Minimal D1 stub for the entitlement reads/writes executeGatewayFetch - * performs (account reverse-resolution `first()` and the conditional - * counter upsert `run()`); secret resolution itself is spied at the - * service layer. Both statements must bind the acting userId — the stub - * fails loudly when a query drops the per-user scoping. - */ +const userMeter = createInMemoryUserMeterEnv() const env = { APP_DB: { prepare(query: string) { @@ -57,9 +52,10 @@ const env = { } }, } as unknown as D1Database, + ...userMeter.env, COOKIE_SECRET: 'test-cookie-secret', SECRET_STORE_KEY: 'test-secret-store-key-32-chars-minimum', -} +} as unknown as Env const props = { baseUrl: 'https://example.com', @@ -626,7 +622,8 @@ test('gateway fetch records outbound_fetch usage metering', async () => { }) expect(successResponse.status).toBe(200) expect(fetchStub).toHaveBeenCalledTimes(1) - expect(waitUntil).toHaveBeenCalledTimes(1) + // waitUntil: entitlement D1 mirror + usage metering + expect(waitUntil).toHaveBeenCalledTimes(2) expect(recordUsageSpy).toHaveBeenCalledTimes(1) expect(recordUsageSpy).toHaveBeenCalledWith(env, { userId: 'user-123', @@ -750,7 +747,8 @@ test('gateway fetch records outbound_fetch usage metering', async () => { waitUntil, }), ).rejects.toThrow('network failed with waitUntil') - expect(waitUntil).toHaveBeenCalledTimes(1) + // waitUntil: entitlement D1 mirror (before fetch) + usage metering (error path) + expect(waitUntil).toHaveBeenCalledTimes(2) expect(recordUsageSpy).toHaveBeenCalledTimes(1) expect(recordUsageSpy.mock.calls[0]?.[1]).toMatchObject({ entityId: 'api.example.com', diff --git a/packages/worker/src/mcp/fetch-gateway.ts b/packages/worker/src/mcp/fetch-gateway.ts index c2450f366f..c76041cd22 100644 --- a/packages/worker/src/mcp/fetch-gateway.ts +++ b/packages/worker/src/mcp/fetch-gateway.ts @@ -87,7 +87,7 @@ function applyOutboundFetchTimeout( } export async function executeGatewayFetch(input: { - env: Pick & UsageEnv + env: Pick & UsageEnv props: FetchGatewayProps request: Request globalFetch?: typeof fetch @@ -131,9 +131,11 @@ export async function executeGatewayFetch(input: { null await consumeDailyEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.props.userId, email, resource: 'outbound_fetches_per_day', + waitUntil: input.waitUntil, }) } const transformed = await expandSecretPlaceholders({ diff --git a/packages/worker/src/mcp/fetch-gateway.workers.test.ts b/packages/worker/src/mcp/fetch-gateway.workers.test.ts index 70a8678c0d..9fa02cee28 100644 --- a/packages/worker/src/mcp/fetch-gateway.workers.test.ts +++ b/packages/worker/src/mcp/fetch-gateway.workers.test.ts @@ -1,15 +1,42 @@ import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' import { expect, test } from 'vitest' import { utcDayKey } from '@kody-internal/shared/date-keys.ts' import { executeGatewayFetch } from '#mcp/fetch-gateway.ts' import { ensureEntitlementTestSchema } from '#worker/entitlements/test-schema.ts' import { isEntitlementLimitError } from '#worker/entitlements/errors.ts' import { planLimits } from '#worker/entitlements/plans.ts' -import { incrementDailyEntitlementCounter } from '#worker/entitlements/service.ts' +import { UserMeter } from '#worker/entitlements/user-meter-do.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' import { seedAccount } from '#worker/test-support/workers-seed.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' +async function seedOutboundFetchCounter(userId: string, count: number) { + const day = utcDayKey() + const updatedAt = new Date().toISOString() + const stub = env.USER_METER.get( + env.USER_METER.idFromName(userMeterDurableObjectName(userId)), + ) + await runInDurableObject(stub, async (instance: UserMeter, state) => { + expect(instance).toBeInstanceOf(UserMeter) + await instance.read({ resource: 'outbound_fetches_per_day', day }) + state.storage.sql.exec( + `INSERT INTO daily_counters (resource, day, count, revision, updated_at) + VALUES (?, ?, ?, 1, ?) + ON CONFLICT(resource, day) DO UPDATE SET + count = excluded.count, + revision = excluded.revision, + updated_at = excluded.updated_at`, + 'outbound_fetches_per_day', + day, + count, + updatedAt, + ) + }) +} + test('gateway fetches consume the daily outbound-fetch entitlement and deny over the plan limit', async () => { silenceIncidentalRuntimeWarnings() await ensureEntitlementTestSchema(env.APP_DB) @@ -30,34 +57,38 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over } const globalFetch = (async () => new Response('ok')) as unknown as typeof fetch + const waitUntilTasks: Array> = [] + const waitUntil = (promise: Promise) => { + waitUntilTasks.push(promise) + } const allowed = await executeGatewayFetch({ env, props, request: new Request('https://api.example.net/data'), globalFetch, + waitUntil, }) expect(await allowed.text()).toBe('ok') - const counter = await env.APP_DB.prepare( - `SELECT count FROM entitlement_daily_counters - WHERE user_id = ? AND resource = 'outbound_fetches_per_day' AND day = ?`, - ) - .bind(userId, utcDayKey()) - .first<{ count: number }>() - expect(Number(counter?.count)).toBe(1) + await Promise.all(waitUntilTasks) + expect( + await userMeterRpc({ env, userId }).read({ + resource: 'outbound_fetches_per_day', + day: utcDayKey(), + }), + ).toMatchObject({ outcome: 'ready', count: 1 }) - // Fill the remaining free-plan quota, then expect denial. - await incrementDailyEntitlementCounter({ - db: env.APP_DB, + // Fill the remaining free-plan quota in UserMeter, then expect denial. + await seedOutboundFetchCounter( userId, - resource: 'outbound_fetches_per_day', - amount: planLimits.free.maxOutboundFetchesPerDay - 1, - }) + planLimits.free.maxOutboundFetchesPerDay, + ) const denied = await executeGatewayFetch({ env, props, request: new Request('https://api.example.net/data'), globalFetch, + waitUntil, }).catch((error: unknown) => error) expect(isEntitlementLimitError(denied)).toBe(true) @@ -70,6 +101,7 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over props: { ...props, email: null }, request: new Request('https://api.example.net/data'), globalFetch, + waitUntil, }).catch((error: unknown) => error) expect(isEntitlementLimitError(deniedWithoutEmail)).toBe(true) @@ -80,6 +112,7 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over props: { ...props, userId: null, email: null }, request: new Request('https://api.example.net/data'), globalFetch, + waitUntil, }) expect(await contextless.text()).toBe('ok') -}) +}, 30_000) diff --git a/packages/worker/src/mcp/tools/execute.ts b/packages/worker/src/mcp/tools/execute.ts index 6105080192..22e16ae25c 100644 --- a/packages/worker/src/mcp/tools/execute.ts +++ b/packages/worker/src/mcp/tools/execute.ts @@ -262,9 +262,11 @@ export async function registerExecuteTool(agent: McpRegistrationAgent) { if (callerContext.user?.userId) { await consumeDailyEntitlement({ db: env.APP_DB, + env, userId: callerContext.user.userId, email: callerContext.user.email, resource: 'execute_calls_per_day', + waitUntil: agent.waitUntil?.bind(agent), }) } diff --git a/packages/worker/src/test-support/user-meter.ts b/packages/worker/src/test-support/user-meter.ts new file mode 100644 index 0000000000..bf3a390f23 --- /dev/null +++ b/packages/worker/src/test-support/user-meter.ts @@ -0,0 +1,182 @@ +import { + isDailyEntitlementResource, + userMeterMirrorUpdatedAtToken, + type DailyEntitlementResource, +} from '#worker/entitlements/user-meter-do.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' + +type MeterRow = { count: number; revision: number } + +/** + * In-memory UserMeter stub keyed by `idFromName` (stable userId) for node-unit + * coverage of expand-phase service/usage paths. Workers suites exercise the + * real Durable Object binding instead. + */ +export function createInMemoryUserMeterEnv() { + const metersByUser = new Map>() + + function counterKey(resource: string, day: string) { + return `${resource}\0${day}` + } + + function meterFor(userId: string) { + const existingRows = metersByUser.get(userId) + const rows = existingRows ?? new Map() + if (!existingRows) metersByUser.set(userId, rows) + + function readRow(resource: string, day: string) { + return rows.get(counterKey(resource, day)) ?? null + } + + function ready(row: MeterRow) { + return { + outcome: 'ready' as const, + count: row.count, + revision: row.revision, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(row.revision), + } + } + + return { + async initialize(input: { + resource: string + day: string + count: number + updatedAt: string + }) { + if (!isDailyEntitlementResource(input.resource)) { + throw new Error(`Invalid daily resource: ${input.resource}`) + } + const key = counterKey(input.resource, input.day) + const existing = rows.get(key) + if (existing) return { ...ready(existing), created: false } + const row = { + count: Math.max(0, Math.trunc(input.count) || 0), + revision: 1, + } + rows.set(key, row) + return { ...ready(row), created: true } + }, + async consume(input: { + resource: string + day: string + limit: number + updatedAt: string + }) { + if (!isDailyEntitlementResource(input.resource)) { + throw new Error(`Invalid daily resource: ${input.resource}`) + } + const resource: DailyEntitlementResource = input.resource + const existing = readRow(resource, input.day) + if (!existing) return { outcome: 'needs_bootstrap' as const } + if (input.limit < 1 || existing.count + 1 > input.limit) { + return { ...ready(existing), consumed: false } + } + const next = { + count: existing.count + 1, + revision: existing.revision + 1, + } + rows.set(counterKey(resource, input.day), next) + return { ...ready(next), consumed: true } + }, + async read(input: { resource: string; day: string }) { + const existing = readRow(input.resource, input.day) + if (!existing) return { outcome: 'needs_bootstrap' as const } + return ready(existing) + }, + async refund(input: { + resource: string + day: string + updatedAt: string + }) { + const existing = readRow(input.resource, input.day) + if (!existing) { + return { + outcome: 'ready' as const, + count: 0, + revision: 0, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(0), + } + } + const next = { + count: Math.max(0, existing.count - 1), + revision: existing.revision + 1, + } + rows.set(counterKey(input.resource, input.day), next) + return ready(next) + }, + async purge() { + rows.clear() + return { ok: true as const } + }, + async exportCounters() { + const counters = [...rows.entries()].map(([entryKey, row]) => { + const [resource, day] = entryKey.split('\0') + return { + resource: resource as DailyEntitlementResource, + day: day!, + count: row.count, + revision: row.revision, + updatedAt: new Date().toISOString(), + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(row.revision), + } + }) + return { counters, nextStartAfter: null, truncated: false } + }, + } + } + + const env = { + USER_METER: { + idFromName: (name: string) => ({ name, toString: () => name }), + get: (id: { name: string }) => meterFor(id.name), + }, + } as unknown as UserMeterEnv + + return { + env, + metersByUser, + async seed(input: { + userId: string + resource: DailyEntitlementResource + day: string + count: number + }) { + await meterFor(input.userId).initialize({ + resource: input.resource, + day: input.day, + count: input.count, + updatedAt: new Date().toISOString(), + }) + }, + } +} + +export function createWaitUntilDrain() { + const tasks: Array> = [] + return { + waitUntil(promise: Promise) { + tasks.push(promise) + }, + async drain() { + await Promise.all(tasks) + tasks.length = 0 + }, + } +} + +/** + * Patch `db.prepare` and restore it via `using` even when the body throws. + */ +export function withPatchedDbPrepare( + db: D1Database, + patch: (originalPrepare: D1Database['prepare']) => D1Database['prepare'], +) { + const originalPrepare = db.prepare.bind(db) + db.prepare = patch(originalPrepare) + return { + [Symbol.dispose]() { + db.prepare = originalPrepare + }, + } +} diff --git a/packages/worker/src/user-scoped-durable-object-name.node.test.ts b/packages/worker/src/user-scoped-durable-object-name.node.test.ts index 12525e9749..c21190f1f2 100644 --- a/packages/worker/src/user-scoped-durable-object-name.node.test.ts +++ b/packages/worker/src/user-scoped-durable-object-name.node.test.ts @@ -9,7 +9,9 @@ import { packageRealtimeSessionDurableObjectName, packageServiceInstanceDurableObjectName, repoSessionDurableObjectName, + runLogDurableObjectName, storageRunnerDurableObjectName, + userMeterDurableObjectName, } from './user-scoped-durable-object-name.ts' const identityModuleRelativePath = @@ -22,6 +24,13 @@ test('user-scoped Durable Object name helpers preserve frozen idFromName contrac // JobManager historically does not trim; keep that wire format frozen. expect(jobManagerDurableObjectName(' user-aaa ')).toBe(' user-aaa ') + expect(runLogDurableObjectName('user-aaa')).toBe('user-aaa') + expect(runLogDurableObjectName(' user-aaa ')).toBe(' user-aaa ') + + expect(userMeterDurableObjectName('user-aaa')).toBe('user-aaa') + // UserMeter matches RunLog: untrimmed userId is the frozen idFromName. + expect(userMeterDurableObjectName(' user-aaa ')).toBe(' user-aaa ') + expect(mcpClientHubDurableObjectName(' user-aaa ')).toBe('user-aaa') expect(storageRunnerDurableObjectName('user-aaa', 'job:1')).toBe( diff --git a/packages/worker/src/user-scoped-durable-object-name.ts b/packages/worker/src/user-scoped-durable-object-name.ts index 69383ea8eb..6fa7e52a20 100644 --- a/packages/worker/src/user-scoped-durable-object-name.ts +++ b/packages/worker/src/user-scoped-durable-object-name.ts @@ -30,6 +30,11 @@ export function runLogDurableObjectName(userId: string) { return userId } +/** UserMeter — one daily-entitlement meter DO per user (untrimmed, like RunLog). */ +export function userMeterDurableObjectName(userId: string) { + return userId +} + /** McpClientHub — one client-hub DO per user (trimmed). */ export function mcpClientHubDurableObjectName(userId: string) { return userId.trim() diff --git a/packages/worker/worker-configuration.d.ts b/packages/worker/worker-configuration.d.ts index f1818a4947..4b6cb5eaae 100644 --- a/packages/worker/worker-configuration.d.ts +++ b/packages/worker/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --config=packages/worker/wrangler.jsonc --env=production ./packages/worker/worker-configuration.d.ts` (hash: c94d594932ed070991b8178999e5f795) +// Generated by Wrangler by running `wrangler types --config=packages/worker/wrangler.jsonc --env=production ./packages/worker/worker-configuration.d.ts` (hash: b7b4d993601fc627f26550ebd7901b3a) // Runtime types generated with workerd@1.20260701.1 2026-04-16 global_fetch_strictly_public,nodejs_compat interface __BaseEnv_Env { OAUTH_KV: KVNamespace; @@ -13,11 +13,13 @@ interface __BaseEnv_Env { FLAG_EXPOSURES: AnalyticsEngineDataset; PLATFORM_FEEDBACK_DISPATCH_QUEUE: Queue; COMMUNITY_ACTIVITY_DISPATCH_QUEUE: Queue; + AUTH_RATE_LIMITER: RateLimit; LOADER: WorkerLoader; APP_LOADER: WorkerLoader; AI: Ai; ASSETS: Fetcher; SENTRY_ENVIRONMENT: "production"; + SIGNUP_MODE: "invite"; SENTRY_TRACES_SAMPLE_RATE: 0; ARTIFACTS_NAMESPACE: "production"; PACKAGE_APP_BASE_URL: "https://kodyapps.dev"; @@ -42,6 +44,7 @@ interface __BaseEnv_Env { JOB_MANAGER: DurableObjectNamespace; STORAGE_RUNNER: DurableObjectNamespace; RUN_LOG: DurableObjectNamespace; + USER_METER: DurableObjectNamespace; REPO_SESSION: DurableObjectNamespace; PACKAGE_REALTIME_SESSION: DurableObjectNamespace; PACKAGE_SERVICE_INSTANCE: DurableObjectNamespace; @@ -50,7 +53,7 @@ interface __BaseEnv_Env { declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/index"); - durableNamespaces: "MCP" | "JobManager" | "StorageRunner" | "RepoSession" | "PackageRealtimeSession" | "PackageServiceInstance" | "RemoteConnectorSession" | "McpClientHub" | "OAuthPurgeCoordinator" | "RunLog"; + durableNamespaces: "MCP" | "JobManager" | "StorageRunner" | "RepoSession" | "PackageRealtimeSession" | "PackageServiceInstance" | "RemoteConnectorSession" | "McpClientHub" | "OAuthPurgeCoordinator" | "RunLog" | "UserMeter"; } interface Env extends __BaseEnv_Env {} } @@ -59,7 +62,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.md" { const value: string; diff --git a/packages/worker/wrangler.jsonc b/packages/worker/wrangler.jsonc index c3d1959989..36bc9f852c 100644 --- a/packages/worker/wrangler.jsonc +++ b/packages/worker/wrangler.jsonc @@ -91,6 +91,10 @@ "new_sqlite_classes": ["RunLog"], "tag": "v20", }, + { + "new_sqlite_classes": ["UserMeter"], + "tag": "v21", + }, ], "observability": { @@ -193,6 +197,10 @@ "class_name": "RunLog", "name": "RUN_LOG", }, + { + "class_name": "UserMeter", + "name": "USER_METER", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", @@ -407,6 +415,10 @@ "class_name": "RunLog", "name": "RUN_LOG", }, + { + "class_name": "UserMeter", + "name": "USER_METER", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", @@ -538,6 +550,10 @@ "class_name": "RunLog", "name": "RUN_LOG", }, + { + "class_name": "UserMeter", + "name": "USER_METER", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", From 8ebf663dff1ef9f0da4006e4d1487c492bf3be96 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 00:11:39 +0000 Subject: [PATCH 04/15] feat(account): inventory UserMeter lifecycle Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 94 ++++++++++-- .../contributing/architecture/entitlements.md | 120 ++++++++++++--- .../contributing/architecture/primitives.yaml | 14 ++ .../worker/src/account/export.node.test.ts | 124 +++++++++++++++ packages/worker/src/account/export.ts | 145 ++++++++++++++---- .../account/user-owned-surfaces.node.test.ts | 14 ++ .../worker/src/account/user-owned-surfaces.ts | 9 ++ .../src/app/account-deletion.node.test.ts | 41 +++++ packages/worker/src/app/account-deletion.ts | 30 ++++ 9 files changed, 530 insertions(+), 61 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 8b2b37a6cd..353aa888ca 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -78,12 +78,13 @@ Deletion must cover these user-owned surfaces: references a stale column. - **Durable Objects:** `JobManager`, `StorageRunner`, `RepoSession`, `RemoteConnectorSession`, `PackageRealtimeSession`, `PackageServiceInstance`, - `McpClientHub`, and `RunLog` are purged through account-deletion RPCs after - their D1 identifiers are collected (`RunLog` is one object per user and needs - no D1 id scan). `MCP` objects remain SDK session-keyed, while - `mcp_agent_sessions` indexes each Durable Object id by authenticated stable - user id so account deletion can purge stored props, conversation state, - raw-fetch state, and transport storage before revoking OAuth grants. + `McpClientHub`, `RunLog`, and `UserMeter` are purged through account-deletion + RPCs after their D1 identifiers are collected (`RunLog` and `UserMeter` are + one object per user and need no D1 id scan). `MCP` objects remain SDK + session-keyed, while `mcp_agent_sessions` indexes each Durable Object id by + authenticated stable user id so account deletion can purge stored props, + conversation state, raw-fetch state, and transport storage before revoking + OAuth grants. - **Vectorize:** memory, job, and saved-package vector ids are derived from D1 rows and removed with `deleteByIds`. - **R2:** raw email MIME blobs in `EMAIL_BLOBS` are enumerated from @@ -164,13 +165,14 @@ migration-safe chunked interface: with `section: "d1_table"` and a table name. Durable storage buckets are read with `section: "storage_runner"` and a `storage_id`, using the same StorageRunner `exportStorage({ pageSize, startAfter })` RPC as the dedicated - storage export capability. R2 raw MIME, attachment, avatar, and icon objects - use `section: "r2_object"`; each response contains at most one 256 KiB base64 - chunk and an opaque cursor. Each request uses bounded `LIMIT 1` ownership - queries rather than reconstructing inventory. Continuation cursors bind the - source row, object key, size, and ETag; ownership/key mutations and object - overwrites are reported instead of mixing generations. Missing objects are - represented explicitly. + storage export capability. User meter counters use `section: "user_meter"` and + the `UserMeter.exportCounters` RPC. R2 raw MIME, attachment, avatar, and icon + objects use `section: "r2_object"`; each response contains at most one 256 KiB + base64 chunk and an opaque cursor. Each request uses bounded `LIMIT 1` + ownership queries rather than reconstructing inventory. Continuation cursors + bind the source row, object key, size, and ETag; ownership/key mutations and + object overwrites are reported instead of mixing generations. Missing objects + are represented explicitly. D1 manifest counts use bounded SQL `COUNT(*)` queries. D1 section rows are read with SQL-level keyset pagination: every query orders by the table's `rowid`, @@ -189,6 +191,11 @@ Durable Object export behavior: section (`exportRuns` RPC; one cursor pages runs first, then ledger rows). Retention is self-enforced inside the DO (~30 days / 2,000 runs; ledger terminal rows 90 days); see [Run records](./run-records.md). +- `UserMeter` exports daily entitlement counter rows through the `user_meter` + section (`exportCounters` RPC; keyset pagination by UTC `day` and `resource`). + Retention is self-enforced inside the DO (seven UTC days of counter and + inbound-delivery-claim rows); see + [Entitlements](./entitlements.md#user-meter-expand-phase). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -445,6 +452,55 @@ Storage split: due jobs - `StorageRunner` SQLite: isolated durable state addressed by `storageId` +## Durable Objects (`UserMeter`) + +Daily rate-style entitlement counters and inbound email delivery-id idempotency +live in a per-user `UserMeter` Durable Object with SQLite +(`packages/worker/src/entitlements/user-meter-do.ts`). The Worker binding is +`USER_METER` (class `UserMeter`; Wrangler SQLite migration tag `v21` via +`new_sqlite_classes` in `packages/worker/wrangler.jsonc`). + +Naming matches `RunLog` and `JobManager`: one object per untrimmed stable MCP +`userId` via `userMeterDurableObjectName(userId)` → `idFromName(userId)` in +`packages/worker/src/user-scoped-durable-object-name.ts`. There is no `user_id` +column inside the DO because the object identity is the user. + +SQLite ownership (schema version tracked in `user_meter_meta`): + +- `daily_counters` — authoritative UTC-day counters for `email_sends_per_day`, + `email_receives_per_day`, `execute_calls_per_day`, and + `outbound_fetches_per_day` (`resource`, `day`, `count`, monotonic `revision`, + `updated_at`). +- `inbound_delivery_claims` — idempotency ledger keyed by inbound `delivery_id` + (scoped by DO identity, so the primary key is delivery id alone). Records the + claim's resource/day, post-charge counter, revision, and `claimed_at` so + Cloudflare Email Routing retries inside the 48-hour inbound dedupe window + cannot double-charge `email_receives_per_day`. + +Retention is self-enforced inside the DO: every read/write path +opportunistically deletes counter and claim rows older than seven UTC days +(`userMeterDailyCounterRetentionDays`). Enforcement only needs the current day; +the window covers timezone edge cases, recent account exports, and inbound +retries. + +**Expand-phase D1 mirror:** enforcement and point reads are authoritative in +UserMeter. D1 `entitlement_daily_counters` is **not** dropped — it remains a +best-effort mirror for existing readers and reporting. After each DO +consume/refund/inbound claim, the entitlements service schedules a non-awaited +absolute mirror write keyed by `(user_id, resource, day)` with a +revision-ordered `updated_at` token (`r/` + zero-padded revision from +`userMeterMirrorUpdatedAtToken`) so late writes cannot overwrite newer state. +See [Entitlements](./entitlements.md#user-meter-expand-phase). + +**Cold bootstrap:** a missing `(resource, day)` row returns `needs_bootstrap`. +The service performs one legacy D1 point read on `entitlement_daily_counters`, +then `initialize()` seeds the DO row with `INSERT OR IGNORE` (concurrent callers +cannot double-apply the baseline). Warm paths never read D1 for enforcement. + +Account deletion calls `UserMeter.purge()` (one RPC per user, no D1 id scan). +Account export pages `UserMeter.exportCounters` through the `user_meter` +manifest section / `account_export_section`. + ### Package state model Saved packages are the only top-level persisted primitive. Their state maps onto @@ -478,6 +534,9 @@ via `durableObjectNameFromParts`); domain helpers such as - `RunLog` — `runLogDurableObjectName(userId)` → `idFromName(userId)`. One execution-history DO per user; there is no `user_id` column inside it because the DO identity is the user. See [Run records](./run-records.md). +- `UserMeter` — `userMeterDurableObjectName(userId)` → `idFromName(userId)`. One + daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`). + See [Entitlements](./entitlements.md#user-meter-expand-phase). - `McpClientHub` — `mcpClientHubDurableObjectName(userId)` → `idFromName(userId.trim())`. - `StorageRunner` — `storageRunnerDurableObjectName(userId, storageId)` → @@ -542,6 +601,8 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` - `JOB_MANAGER` (Durable Objects) - `RUN_LOG` (Durable Objects; per-user run records — see [Run records](./run-records.md)) +- `USER_METER` (Durable Objects; per-user daily entitlement counters — see + [Entitlements](./entitlements.md#user-meter-expand-phase)) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) @@ -786,6 +847,8 @@ to `durableObjectNameFromParts`). - `JobManager`: `idFromName(userId)` (no trim). - `RunLog`: `idFromName(userId)` (no trim); one execution-history DO per user. +- `UserMeter`: `idFromName(userId)` (no trim); one daily-entitlement meter DO + per user. - `McpClientHub`: `idFromName(userId.trim())`. - `StorageRunner`: `idFromName(JSON.stringify([userId, storageId]))`. - `RepoSession`: `idFromName(repo_sessions.id)`; the key is not user-prefixed, @@ -953,7 +1016,10 @@ Current retention policies: blob delete fails, those rows are skipped and still selected on later runs so cleanup can retry. Dependent `email_attachments` rows are deleted before their messages, and threads left with no messages are pruned for the affected users. -- `entitlement_daily_counters`: daily rate counters keep 400 days by `day` key. +- `entitlement_daily_counters`: expand-phase **mirror** of UserMeter daily + counters (authoritative state lives in the per-user `UserMeter` DO). Rows keep + 400 days by `day` key until mirror retirement is verified after + reporting-off-D1 merges; the table is not dropped in this phase. - `usage_rollups`: per user/metric/month rollups keep 24 months by `month` key; raw Analytics Engine usage events follow platform retention. - `feature_flag_exposure_rollups`: local-dev/test flag exposure rollups keep 90 diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index 5b66773a99..5bee0a0a6f 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -107,7 +107,68 @@ two compute surfaces `usage-metering.md` already observes: cannot grant elevated quotas. Both consume only when the context has a `userId`, matching the usage-metering -rule that events without an owning user are skipped. +rule that events without an owning user are skipped. Daily consumption is +authoritative in the per-user `UserMeter` Durable Object; see +[UserMeter (expand phase)](#user-meter-expand-phase). + +## UserMeter (expand phase) + +Daily rate-style resources (`email_sends_per_day`, `email_receives_per_day`, +`execute_calls_per_day`, `outbound_fetches_per_day`) are **authoritative in the +per-user `UserMeter` Durable Object** (`USER_METER` binding). Code lives in +`packages/worker/src/entitlements/user-meter-do.ts` and `user-meter-client.ts`; +storage layout and naming are documented in [Data storage](./data-storage.md). + +**Strong enforcement:** `consumeDailyEntitlement` and inbound +`consumeInboundDelivery` RPCs check the plan limit and increment inside the DO. +The Durable Object request model serializes mutations per user; counter updates +use optimistic concurrency on monotonic `revision` so concurrent consumes cannot +overshoot. Missing `(resource, day)` rows return `needs_bootstrap` rather than +silently starting at zero on a warm account. + +**Cold bootstrap:** on `needs_bootstrap`, the service performs one legacy D1 +point read on `entitlement_daily_counters`, then `UserMeter.initialize()` seeds +the row with `INSERT OR IGNORE` (concurrent callers cannot double-apply the +baseline). The warm enforcement path awaits only the DO RPC — never APP_DB. + +**Non-awaited D1 mirror:** after each successful consume, refund, or inbound +delivery claim, the service schedules a best-effort absolute mirror write to +`entitlement_daily_counters` via `waitUntil` when available (otherwise a caught +void promise). Mirror ordering uses the DO-minted `mirrorUpdatedAt` token +(`r/` + zero-padded revision) in the existing `updated_at` TEXT column so late +writes cannot overwrite newer state, including refunds that lower `count`. +Mirror failures are logged and never affect enforcement. The D1 table is **not** +dropped in this phase — it remains for existing readers and reporting. + +**Point-read surfaces** now call `readDailyEntitlementResourceUsage` (UserMeter +with the same cold-bootstrap path) instead of reading D1 directly: + +- Account usage UI — `packages/worker/src/app/account-usage-data.ts` +- `email_usage_get` MCP capability +- Admin per-user usage drill-down — + `packages/worker/src/admin/user-usage-data.ts` + +Non-daily resources and contexts without `USER_METER` still use +`readEntitlementResourceUsage` against D1. + +**Inbound retry idempotency:** inbound receive quota uses +`UserMeter.consumeInboundDelivery`, which atomically claims `delivery_id` and +consumes one `email_receives_per_day` unit inside a SQLite transaction. Retries +return the accepted counter without incrementing (`replayed: true`). +Cross-UTC-day retries use the original claim's resource/day. The legacy D1 +mirror is scheduled from the email path via +`scheduleAbsoluteDailyEntitlementMirror` so the email subsystem does not +duplicate mirror SQL. + +**Remaining migration dependency:** retiring the D1 mirror and +`entitlement_daily_counters` table waits until reporting-off-D1 work merges and +mirror parity is verified in production. This expand phase intentionally does +**not** move `storage_bytes` enforcement, `package_service_states` running +counts, or account-deletion write fencing to UserMeter. + +Module wiring: `consumeDailyEntitlement`, `refundDailyEntitlement`, and +`readDailyEntitlementResourceUsage` in `service.ts` require `env.USER_METER` and +fail closed when the binding is missing. ## Schema history @@ -245,21 +306,35 @@ Rules: package services are counted from `package_service_states` (status `running` and freshly heartbeaten), not from run-history rows — see [Run records](./run-records.md) (`state-vs-history`). -- **Rate-style limits** (email sends and receives per day) use the - `entitlement_daily_counters` table keyed by `(user_id, resource, day)` with - UTC day keys. Call `consumeDailyEntitlement` on every attempt: it checks the - plan limit from `resolvePlanLimit` and increments the counter in one - conditional D1 upsert (no check-then-increment race). Every resolved plan has - a finite numeric limit. Counting attempts rather than successes keeps the - limit abuse-resistant for permanent rejects (parse failures, entitlement/quota - rejects). Only typed pre-commit `RetryableInboundStorageError` failures - (thread prework, R2 put, D1 message/attachment storage after successful - cleanup) refund exactly one `email_receives_per_day` unit via - `refundDailyEntitlement` for the same UTC day that was charged, so Cloudflare - Email Routing retries do not burn the daily receive quota. Post-commit - bookkeeping failures do not refund or retry. - `incrementDailyEntitlementCounter` remains for raw counter writes (tests, - backfills). +- **Rate-style limits** (email sends/receives per day, execute calls per day, + outbound fetches per day) are **authoritative in the per-user UserMeter + Durable Object** (UTC day keys). Call `consumeDailyEntitlement` on every + attempt: it resolves the plan limit, atomically checks and increments inside + the DO, and throws `EntitlementLimitError` when over limit. Every resolved + plan has a finite numeric limit. Counting attempts rather than successes keeps + the limit abuse-resistant for permanent rejects (parse failures, + entitlement/quota rejects). + + **Cold bootstrap:** missing `(resource, day)` rows trigger one legacy D1 point + read and a single `UserMeter.initialize()` before retrying the consume. + + **D1 mirror (expand phase):** after each successful consume/refund/inbound + claim, a best-effort, non-awaited absolute mirror write updates + `entitlement_daily_counters` with revision-ordered `updated_at` tokens. The + table is not dropped — it remains for existing readers and reporting until + reporting-off-D1 retirement is verified. + + Only typed pre-commit `RetryableInboundStorageError` failures (thread prework, + R2 put, D1 message/attachment storage after successful cleanup) refund exactly + one `email_receives_per_day` unit via `refundDailyEntitlement` for the same + UTC day that was charged, so Cloudflare Email Routing retries do not burn the + daily receive quota. Post-commit bookkeeping failures do not refund or retry. + Inbound routing retries that replay the same `delivery_id` are idempotent via + `UserMeter.consumeInboundDelivery` and do not consume an additional unit. + + `incrementDailyEntitlementCounter` remains for raw D1 counter writes (tests, + backfills, and legacy paths). + - **Boolean allowances** (persistent package services) are modeled as limit `0` (not allowed) vs `1` (allowed) so the numeric contract stays uniform. - **Per-unit size limits** (`email_message_bytes`) compare one candidate value @@ -328,9 +403,9 @@ separate statements, so a burst of concurrent creates can overshoot a limit by a few rows before the next check sees the new count. That is an accepted trade-off — these limits are order-of-magnitude denial-of-wallet caps, not billing-grade accounting, and folding every insert into a conditional statement would couple -the entitlements module to each resource's write path. The rate-style path does -not share this window: `consumeDailyEntitlement` checks and increments in a -single conditional upsert. +the entitlements module to each resource's write path. Daily rate-style limits +do not share this window: UserMeter consumes are serialized per user inside the +DO with revision-checked updates. ## How to add an enforcement point @@ -459,6 +534,9 @@ in [`../environment-variables.md`](../environment-variables.md). `0066-stripe-billing.sql`; owned by `packages/worker/src/billing/`, read by `getUserPlan` via `resolveEffectivePlan`. `stripe_plan` stays nullable because it is Stripe-derived; `max` is manual-only. -- `entitlement_daily_counters` — rate counters, created by migration - `0048-user-plans-and-entitlement-counters.sql`; included in the +- `entitlement_daily_counters` — expand-phase **mirror** of UserMeter daily + counters (authoritative state in the per-user `UserMeter` DO), created by + migration `0048-user-plans-and-entitlement-counters.sql`; included in the account-deletion cascade (`packages/worker/src/app/account-deletion.ts`). + Table retirement waits until reporting-off-D1 merges and mirror parity is + verified. diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index 21d6e45f63..e559d43738 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -487,6 +487,19 @@ primitives: docs: - docs/contributing/architecture/run-records.md + - id: user-meter + group: storage + name: User meter + summary: + Per-user UserMeter DO SQLite for daily entitlement counters and inbound + delivery idempotency. + code: + - packages/worker/src/entitlements/user-meter-do.ts + - packages/worker/src/entitlements/user-meter-client.ts + docs: + - docs/contributing/architecture/data-storage.md + - docs/contributing/architecture/entitlements.md + - id: email-blobs-r2 group: storage name: Email blob storage (R2) @@ -605,3 +618,4 @@ invariants: docs: - docs/contributing/architecture/usage-metering.md - docs/contributing/architecture/run-records.md + - docs/contributing/architecture/entitlements.md diff --git a/packages/worker/src/account/export.node.test.ts b/packages/worker/src/account/export.node.test.ts index d1d741058d..bdcaad8d14 100644 --- a/packages/worker/src/account/export.node.test.ts +++ b/packages/worker/src/account/export.node.test.ts @@ -1474,6 +1474,130 @@ test('account export includes run_records section with runs and log lines', asyn ]) }) +test('account export includes user_meter counters, pages them, and warns on truncation', async () => { + const { sqlite, db } = createMigratedDb() + sqlite.exec(` + INSERT INTO users ( + id, username, email, password_hash, created_at, updated_at, + email_verified_at, stable_user_id + ) + VALUES ( + 1, 'user-a', 'a@example.com', 'password-hash-a', '2026-07-05', + '2026-07-05', '2026-07-05', 'user-aaa' + ); + `) + const counters = [ + { + resource: 'email_sends_per_day' as const, + day: '2026-07-30', + count: 2, + revision: 2, + updatedAt: '2026-07-30T01:00:00.000Z', + mirrorUpdatedAt: 'r/00000000000000000002', + }, + { + resource: 'execute_calls_per_day' as const, + day: '2026-07-30', + count: 5, + revision: 5, + updatedAt: '2026-07-30T02:00:00.000Z', + mirrorUpdatedAt: 'r/00000000000000000005', + }, + { + resource: 'outbound_fetches_per_day' as const, + day: '2026-07-31', + count: 1, + revision: 1, + updatedAt: '2026-07-31T00:00:00.000Z', + mirrorUpdatedAt: 'r/00000000000000000001', + }, + ] + const exportCounters = vi.fn( + async (input: { pageSize?: number; startAfter?: string | null }) => { + const pageSize = input.pageSize ?? 100 + const startIndex = input.startAfter + ? counters.findIndex( + (row) => `${row.day}:${row.resource}` === input.startAfter, + ) + 1 + : 0 + const page = counters.slice(startIndex, startIndex + pageSize) + const truncated = startIndex + pageSize < counters.length + return { + counters: page, + nextStartAfter: truncated + ? `${page.at(-1)!.day}:${page.at(-1)!.resource}` + : null, + truncated, + } + }, + ) + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const env = { + APP_DB: db, + USER_METER: { + idFromName, + get: () => ({ exportCounters }), + }, + } as unknown as Env + + const accountExport = await createAccountExport({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + }) + expect(idFromName).toHaveBeenCalledWith('user-aaa') + expect(accountExport.manifest.sections.user_meter?.count).toBe(3) + expect(accountExport.durableObjects.userMeter).toEqual({ + counters, + nextStartAfter: null, + truncated: false, + }) + expect(accountExport.manifest.warnings).not.toEqual( + expect.arrayContaining([ + expect.stringContaining('entitlement_daily_counters'), + ]), + ) + + const first = await readAccountExportSection({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + section: 'user_meter', + pageSize: 2, + }) + expect(first.items).toEqual(counters.slice(0, 2)) + expect(first.truncated).toBe(true) + expect(first.nextStartAfter).toBe('2026-07-30:execute_calls_per_day') + + const second = await readAccountExportSection({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + section: 'user_meter', + pageSize: 2, + startAfter: first.nextStartAfter ?? undefined, + }) + expect(second.items).toEqual(counters.slice(2)) + expect(second.truncated).toBe(false) + expect(second.nextStartAfter).toBeNull() + expect(exportCounters).toHaveBeenCalled() + + exportCounters.mockImplementation(async () => ({ + counters: [counters[0]!], + nextStartAfter: 'cursor-more', + truncated: true, + })) + const truncatedExport = await createAccountExport({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + }) + expect(truncatedExport.durableObjects.userMeter?.truncated).toBe(true) + expect(truncatedExport.manifest.warnings).toContain( + 'User meter counters were truncated in the full export; use account_export_section with section "user_meter" to retrieve additional pages.', + ) +}) + test('account export includes a package service known only via package_service_states', async () => { consoleWarn.mockImplementation(() => {}) try { diff --git a/packages/worker/src/account/export.ts b/packages/worker/src/account/export.ts index 6fe7f08622..a5874f0911 100644 --- a/packages/worker/src/account/export.ts +++ b/packages/worker/src/account/export.ts @@ -34,6 +34,11 @@ import { listRunRecordStorageIds, summarizeRunRecords, } from '#worker/run-records/service.ts' +import { + userMeterNamespace, + userMeterRpc, +} from '#worker/entitlements/user-meter-client.ts' +import { type UserMeterExportResult } from '#worker/entitlements/user-meter-do.ts' import { resolveUserStableId } from '#worker/user-id.ts' import { listAccountUserPackageServices, @@ -55,6 +60,7 @@ export const accountExportSectionNames = [ 'storage_runner', 'job_manager', 'run_records', + 'user_meter', 'remote_connector_session', 'package_service', 'oauth_grants', @@ -126,6 +132,7 @@ type UserExportInventory = { type ManifestInventoryCounts = { storageRunners: number runRecords: number + userMeterCounters: number remoteConnectorSessions: number packageServices: number artifactRepos: number @@ -198,6 +205,7 @@ export type AccountExportArtifactRepo = { type AccountExportDurableObjects = { jobManager: unknown | null runRecords: Awaited> | null + userMeter: UserMeterExportResult | null remoteConnectorSessions: Array<{ instanceId: string export: RemoteConnectorSessionExport | null @@ -977,6 +985,7 @@ async function collectManifestInventoryCounts(input: { const [ storageRunners, runRecords, + userMeterCounters, remoteConnectorSessions, packageServices, artifactRepos, @@ -993,6 +1002,16 @@ async function collectManifestInventoryCounts(input: { }) return summary.total }), + safeCount('user meter counters', async () => { + if (!userMeterNamespace(input.env)) return 0 + const page = await userMeterRpc({ + env: input.env, + userId: input.userId, + }).exportCounters({ + pageSize: maxExportPageSize, + }) + return page.counters.length + }), safeCount('remote connectors', async () => countScalar( input.env, @@ -1030,6 +1049,7 @@ async function collectManifestInventoryCounts(input: { return { storageRunners, runRecords, + userMeterCounters, remoteConnectorSessions, packageServices, artifactRepos, @@ -1272,6 +1292,36 @@ async function exportUserRunRecords(input: { } } +async function exportUserMeterCounters(input: { + env: AccountExportEnv + userId: string + warnings: Array +}): Promise { + try { + if (!userMeterNamespace(input.env)) { + input.warnings.push( + 'USER_METER binding was unavailable; user meter counters were not exported.', + ) + return null + } + const page = await userMeterRpc({ + env: input.env, + userId: input.userId, + }).exportCounters({ + pageSize: maxExportPageSize, + }) + if (page.truncated) { + input.warnings.push( + `User meter counters were truncated in the full export; use account_export_section with section "user_meter" to retrieve additional pages.`, + ) + } + return page + } catch (error) { + input.warnings.push(`User meter export failed: ${getErrorMessage(error)}`) + return null + } +} + async function exportRemoteConnectorSessions(input: { env: AccountExportEnv userId: string @@ -1364,32 +1414,42 @@ async function exportDurableObjects(input: { inventory: UserExportInventory warnings: Array }): Promise { - const [storageRunners, remoteConnectorSessions, packageServices, runRecords] = - await Promise.all([ - exportStorageRunners({ - env: input.env, - userId: input.userId, - storageIds: input.inventory.storageIds, - warnings: input.warnings, - }), - exportRemoteConnectorSessions({ - env: input.env, - userId: input.userId, - connectors: input.inventory.remoteConnectors, - warnings: input.warnings, - }), - exportPackageServices({ - env: input.env, - userId: input.userId, - services: input.inventory.packageServices, - warnings: input.warnings, - }), - exportUserRunRecords({ - env: input.env, - userId: input.userId, - warnings: input.warnings, - }), - ]) + const [ + storageRunners, + remoteConnectorSessions, + packageServices, + runRecords, + userMeter, + ] = await Promise.all([ + exportStorageRunners({ + env: input.env, + userId: input.userId, + storageIds: input.inventory.storageIds, + warnings: input.warnings, + }), + exportRemoteConnectorSessions({ + env: input.env, + userId: input.userId, + connectors: input.inventory.remoteConnectors, + warnings: input.warnings, + }), + exportPackageServices({ + env: input.env, + userId: input.userId, + services: input.inventory.packageServices, + warnings: input.warnings, + }), + exportUserRunRecords({ + env: input.env, + userId: input.userId, + warnings: input.warnings, + }), + exportUserMeterCounters({ + env: input.env, + userId: input.userId, + warnings: input.warnings, + }), + ]) let jobManager: unknown | null = null try { jobManager = await exportJobManagerForUser({ @@ -1402,6 +1462,7 @@ async function exportDurableObjects(input: { return { jobManager, runRecords, + userMeter, remoteConnectorSessions, packageServices, storageRunners, @@ -1467,6 +1528,17 @@ function buildManifest(input: { ), discovery: { section: 'run_records' }, } + sections.user_meter = { + count: + input.durableObjects?.userMeter == null + ? (input.inventoryCounts?.userMeterCounters ?? 0) + : input.durableObjects.userMeter.counters.length, + warnings: input.warnings.filter( + (warning) => + warning.startsWith('User meter ') || warning.startsWith('USER_METER '), + ), + discovery: { section: 'user_meter' }, + } sections.remote_connector_sessions = { count: input.inventoryCounts?.remoteConnectorSessions ?? @@ -1792,6 +1864,27 @@ export async function readAccountExportSection(input: { warnings, } } + if (input.section === 'user_meter') { + if (!userMeterNamespace(input.env)) { + throw new Error('USER_METER binding was unavailable.') + } + const pageSize = normalizePageSize(input.pageSize) + const page = await userMeterRpc({ + env: input.env, + userId: input.mcpUserId, + }).exportCounters({ + pageSize, + startAfter: input.startAfter ?? null, + }) + return { + section: input.section, + items: page.counters, + truncated: page.truncated, + nextStartAfter: page.nextStartAfter, + pageSize, + warnings, + } + } if (input.section === 'remote_connector_session') { if (!input.instanceId) { throw new Error( diff --git a/packages/worker/src/account/user-owned-surfaces.node.test.ts b/packages/worker/src/account/user-owned-surfaces.node.test.ts index c1a6dff39c..3772f2dc93 100644 --- a/packages/worker/src/account/user-owned-surfaces.node.test.ts +++ b/packages/worker/src/account/user-owned-surfaces.node.test.ts @@ -59,11 +59,25 @@ test('account deletion and export consume the out-of-band surface registry', () ).toBe(true) expect(coverage.r2SurfaceIds.has('email_raw_mime')).toBe(true) expect(coverage.r2SurfaceIds.has('user_avatar')).toBe(true) + expect(coverage.durableObjectIds.has('user_meter')).toBe(true) + expect( + accountUserOwnedDurableObjectSurfaces.find( + (surface) => surface.id === 'user_meter', + ), + ).toMatchObject({ + binding: 'USER_METER', + deletionResultKey: 'userMeters', + export: 'include', + }) expect( accountUserOwnedKvKeySchemes.some((scheme) => scheme.prefixTemplate?.includes('source-snapshot:v1:'), ), ).toBe(true) + expect(accountExportSource).toContain("'user_meter'") + expect(accountExportSource).toContain('userMeterRpc') + expect(accountDeletionSource).toContain('userMeterRpc') + expect(accountDeletionSource).toContain('userMeters') expect(accountDeletionSource).toContain( "from '#worker/account/user-owned-surfaces.ts'", diff --git a/packages/worker/src/account/user-owned-surfaces.ts b/packages/worker/src/account/user-owned-surfaces.ts index 38cce17acb..9975b02533 100644 --- a/packages/worker/src/account/user-owned-surfaces.ts +++ b/packages/worker/src/account/user-owned-surfaces.ts @@ -8,6 +8,7 @@ export type UserOwnedDurableObjectSurface = { | 'package_realtime_session' | 'package_service_instance' | 'run_log' + | 'user_meter' | 'mcp' binding: string /** Result key used in AccountDeletionResult.clearedDurableObjects when purged */ @@ -123,6 +124,14 @@ export const accountUserOwnedDurableObjectSurfaces: ReadonlyArray [] as Array, }), }, + USER_METER: { + idFromName: durableObjectId, + get: () => ({ purge: async () => ({ ok: true as const }) }), + }, JOB_MANAGER: { idFromName: durableObjectId, get: () => ({ purgeUser: async () => ({ ok: true as const }) }), @@ -1106,6 +1110,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async const clearStorageMock = vi.fn(async () => ({ ok: true as const })) const clearRunLogMock = vi.fn(async () => ({ ok: true as const })) + const purgeUserMeterMock = vi.fn(async () => ({ ok: true as const })) const purgeJobManagerMock = vi.fn(async () => ({ ok: true as const })) const purgeRepoSessionMock = vi.fn(async () => ({ ok: true as const })) const purgeRemoteConnectorMock = vi.fn(async () => ({ ok: true as const })) @@ -1131,6 +1136,10 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async listStorageIds: async () => [] as Array, }), }, + USER_METER: { + idFromName: (name: string) => name as unknown as DurableObjectId, + get: () => ({ purge: purgeUserMeterMock }), + }, JOB_MANAGER: { idFromName: (name: string) => name as unknown as DurableObjectId, get: () => ({ purgeUser: purgeJobManagerMock }), @@ -1431,6 +1440,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async expect(result.clearedDurableObjects).toMatchObject({ storageRunners: 6, runLogs: 1, + userMeters: 1, jobManagers: 1, repoSessions: 1, remoteConnectorSessions: 1, @@ -1443,6 +1453,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async packageServiceInstances: 3, }) expect(clearRunLogMock).toHaveBeenCalledTimes(1) + expect(purgeUserMeterMock).toHaveBeenCalledTimes(1) expect(purgeMcpClientHubMock).toHaveBeenCalledTimes(1) expect(purgeMcpAgentSessionMock).toHaveBeenCalledWith({ userId: userAaa, @@ -1990,6 +2001,36 @@ test('account deletion empties the user RunLog DO and leaves other users untouch expect(clearedStorageIds.some((id) => id.includes('bbb-bucket'))).toBe(false) }) +test('account deletion reports a missing USER_METER binding and remains retryable', async () => { + const { db, rows } = createTestDb({ + users: [{ id: 1, email: 'a@example.com', stable_user_id: 'user-aaa' }], + }) + await expect( + deleteUserAccount({ + env: createSuccessfulDeletionEnv(db, { + USER_METER: undefined, + }), + dbUserId: 1, + mcpUserId: 'user-aaa', + }), + ).rejects.toMatchObject({ + cleanupErrors: expect.arrayContaining([ + 'USER_METER binding was unavailable; the user meter Durable Object was not purged.', + ]), + partialResult: expect.objectContaining({ + clearedDurableObjects: expect.objectContaining({ + userMeters: 0, + }), + }), + }) + expect(rows.users).toEqual([ + expect.objectContaining({ + id: 1, + deleting_at: expect.any(String), + }), + ]) +}) + test('account deletion purges a StorageRunner known only via user_storage_buckets', async () => { const userId = 'user-bucket-only' const clearStorage = vi.fn(async () => ({ ok: true as const })) diff --git a/packages/worker/src/app/account-deletion.ts b/packages/worker/src/app/account-deletion.ts index 50ed4395be..bb2d825d7d 100644 --- a/packages/worker/src/app/account-deletion.ts +++ b/packages/worker/src/app/account-deletion.ts @@ -17,6 +17,10 @@ import { mcpClientHubDurableObjectName } from '#worker/user-scoped-durable-objec import { packageServiceRpc } from '#worker/package-runtime/package-service.ts' import { packageRealtimeSessionRpc } from '#worker/package-runtime/realtime-session.ts' import { clearRunRecords } from '#worker/run-records/service.ts' +import { + userMeterNamespace, + userMeterRpc, +} from '#worker/entitlements/user-meter-client.ts' import { listAccountUserPackageServices, listAccountUserStorageIds, @@ -596,6 +600,27 @@ async function clearRunLog(input: { } } +async function purgeUserMeter(input: { + env: Env + userId: string + warnings: Array +}): Promise { + try { + if (!userMeterNamespace(input.env)) { + input.warnings.push( + 'USER_METER binding was unavailable; the user meter Durable Object was not purged.', + ) + return 0 + } + await userMeterRpc({ env: input.env, userId: input.userId }).purge() + return 1 + } catch (error) { + const message = getErrorMessage(error) + input.warnings.push(`User meter purge failed: ${message}`) + return 0 + } +} + async function purgeJobManager(input: { env: Env userId: string @@ -1092,6 +1117,11 @@ export async function deleteUserAccount(input: { userId: input.mcpUserId, warnings, }) + result.clearedDurableObjects.userMeters = await purgeUserMeter({ + env: input.env, + userId: input.mcpUserId, + warnings, + }) if (input.env.BUNDLE_ARTIFACTS_KV) { const sourceSnapshotKeys = await listKvKeysByPrefix({ From 4a60dd992fe172d797fc2084469042efc4cbbbef Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 00:23:10 +0000 Subject: [PATCH 05/15] test(entitlements): provide UserMeter fixtures Co-authored-by: Kent C. Dodds --- .../worker/src/email/inbound.workers.test.ts | 309 +++++++++--------- .../admin/admin-capabilities.node.test.ts | 4 + .../operation-request.node.test.ts | 4 + .../worker/src/mcp/observability.node.test.ts | 4 + .../worker/src/mcp/tools/execute.node.test.ts | 4 + 5 files changed, 174 insertions(+), 151 deletions(-) diff --git a/packages/worker/src/email/inbound.workers.test.ts b/packages/worker/src/email/inbound.workers.test.ts index 7d9ff38520..feadc77b7b 100644 --- a/packages/worker/src/email/inbound.workers.test.ts +++ b/packages/worker/src/email/inbound.workers.test.ts @@ -119,164 +119,171 @@ async function seedVerifiedAccount(input: { // budget these explicitly like the other sandbox-executing suites rather than // letting them flake under a loaded `npm run validate`. const subscriptionDispatchTimeoutMs = 60_000 +// handleInboundEmail touches the real USER_METER DO; isolated runs finish in +// a few seconds but the default 5s budget flakes under full-suite contention. +const inboundRoutingTimeoutMs = 15_000 -test('inbound email routes {username}@platform-domain and auto-provisions the default inbox', async () => { - // Usage recording degrades with a warn when the usage_rollups table is - // not part of this test's schema; that is incidental to routing. - silenceIncidentalRuntimeWarnings() - await ensureEmailTestSchema(env.APP_DB) - const username = `inbound-${crypto.randomUUID().slice(0, 8)}` - const accountEmail = `account-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - const address = `${username}@${platformDomain}` - await seedVerifiedAccount({ - db: env.APP_DB, - email: accountEmail, - username, - }) - - const firstMessage = createForwardableEmailMessage({ - from: 'stranger@example.net', - to: address, - raw: [ - 'From: Stranger ', - `To: ${address}`, - 'Subject: Unknown sender', - 'Message-ID: ', - '', - 'Please help.', - ].join('\r\n'), - }) - await handleInboundEmail(firstMessage, createInboundEnv()) - expect(firstMessage.rejectedReason).toBeNull() +test( + 'inbound email routes {username}@platform-domain and auto-provisions the default inbox', + async () => { + // Usage recording degrades with a warn when the usage_rollups table is + // not part of this test's schema; that is incidental to routing. + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const username = `inbound-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `account-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) - // First inbound provisioned the default inbox and address. - const inboxes = await listEmailInboxesForUser({ db: env.APP_DB, userId }) - expect(inboxes).toHaveLength(1) - expect(inboxes[0]).toMatchObject({ name: defaultEmailInboxName }) - const addresses = await listEmailInboxAddressesForUser({ - db: env.APP_DB, - userId, - }) - expect(addresses).toHaveLength(1) - expect(addresses[0]).toMatchObject({ - address, - localPart: username, - domain: platformDomain, - }) - // Provisioning also created the platform-assigned verified sender - // identity for the same address. - const identity = await env.APP_DB.prepare( - `SELECT email, domain, status FROM email_sender_identities + const firstMessage = createForwardableEmailMessage({ + from: 'stranger@example.net', + to: address, + raw: [ + 'From: Stranger ', + `To: ${address}`, + 'Subject: Unknown sender', + 'Message-ID: ', + '', + 'Please help.', + ].join('\r\n'), + }) + await handleInboundEmail(firstMessage, createInboundEnv()) + expect(firstMessage.rejectedReason).toBeNull() + + // First inbound provisioned the default inbox and address. + const inboxes = await listEmailInboxesForUser({ db: env.APP_DB, userId }) + expect(inboxes).toHaveLength(1) + expect(inboxes[0]).toMatchObject({ name: defaultEmailInboxName }) + const addresses = await listEmailInboxAddressesForUser({ + db: env.APP_DB, + userId, + }) + expect(addresses).toHaveLength(1) + expect(addresses[0]).toMatchObject({ + address, + localPart: username, + domain: platformDomain, + }) + // Provisioning also created the platform-assigned verified sender + // identity for the same address. + const identity = await env.APP_DB.prepare( + `SELECT email, domain, status FROM email_sender_identities WHERE user_id = ? AND email = ?`, - ) - .bind(userId, address) - .first>() - expect(identity).toEqual({ - email: address, - domain: platformDomain, - status: 'verified', - }) - const inbox = inboxes[0]! - - const secondMessage = createForwardableEmailMessage({ - from: 'agent@trusted.example', - to: `${username.toUpperCase()}@${platformDomain}`, - raw: [ - 'From: Agent ', - `To: ${address}`, - 'Subject: Approved sender', - 'Message-ID: ', - '', - 'Approved body.', - ].join('\r\n'), - }) - await handleInboundEmail(secondMessage, createInboundEnv()) - expect(secondMessage.rejectedReason).toBeNull() + ) + .bind(userId, address) + .first>() + expect(identity).toEqual({ + email: address, + domain: platformDomain, + status: 'verified', + }) + const inbox = inboxes[0]! - // The second delivery reuses the provisioned inbox instead of creating - // another one. - expect( - await listEmailInboxesForUser({ db: env.APP_DB, userId }), - ).toHaveLength(1) + const secondMessage = createForwardableEmailMessage({ + from: 'agent@trusted.example', + to: `${username.toUpperCase()}@${platformDomain}`, + raw: [ + 'From: Agent ', + `To: ${address}`, + 'Subject: Approved sender', + 'Message-ID: ', + '', + 'Approved body.', + ].join('\r\n'), + }) + await handleInboundEmail(secondMessage, createInboundEnv()) + expect(secondMessage.rejectedReason).toBeNull() - const messages = await listEmailMessages({ - db: env.APP_DB, - userId, - inboxId: inbox.id, - limit: 10, - }) - expect(messages).toHaveLength(2) - expect(messages[0]).toMatchObject({ - fromAddress: 'agent@trusted.example', - subject: 'Approved sender', - processingStatus: 'stored', - }) - expect(messages[1]).toMatchObject({ - fromAddress: 'stranger@example.net', - subject: 'Unknown sender', - error: null, - }) + // The second delivery reuses the provisioned inbox instead of creating + // another one. + expect( + await listEmailInboxesForUser({ db: env.APP_DB, userId }), + ).toHaveLength(1) - const normalizedExistingThread = await createEmailThread({ - db: env.APP_DB, - userId, - inboxId: inbox.id, - subjectNormalized: 'normalized subject', - }) - const subjectOnlyMessage = createForwardableEmailMessage({ - from: 'sender@example.net', - to: address, - raw: [ - 'From: Sender ', - `To: ${address}`, - 'Subject: Re: Normalized Subject', - '', - 'Subject-only body.', - ].join('\r\n'), - }) - await handleInboundEmail(subjectOnlyMessage, createInboundEnv()) - const subjectOnly = await listEmailMessages({ - db: env.APP_DB, - userId, - inboxId: inbox.id, - limit: 1, - }) - expect(subjectOnly[0]?.threadId).not.toBe(normalizedExistingThread.id) + const messages = await listEmailMessages({ + db: env.APP_DB, + userId, + inboxId: inbox.id, + limit: 10, + }) + expect(messages).toHaveLength(2) + expect(messages[0]).toMatchObject({ + fromAddress: 'agent@trusted.example', + subject: 'Approved sender', + processingStatus: 'stored', + }) + expect(messages[1]).toMatchObject({ + fromAddress: 'stranger@example.net', + subject: 'Unknown sender', + error: null, + }) - // RFC 5233 subaddressing routes {username}+{tag} to the same inbox and - // keeps the full tagged address in the stored to_addresses so package - // handlers can dispatch on the tag. - const taggedAddress = `${username}+billing@${platformDomain}` - const taggedMessage = createForwardableEmailMessage({ - from: 'invoices@example.net', - to: taggedAddress, - raw: [ - 'From: Invoices ', - `To: ${taggedAddress}`, - 'Subject: Subaddressed mail', - 'Message-ID: ', - '', - 'Tagged body.', - ].join('\r\n'), - }) - await handleInboundEmail(taggedMessage, createInboundEnv()) - expect(taggedMessage.rejectedReason).toBeNull() - const taggedStored = await listEmailMessages({ - db: env.APP_DB, - userId, - inboxId: inbox.id, - limit: 1, - }) - expect(taggedStored[0]).toMatchObject({ - subject: 'Subaddressed mail', - toAddresses: [taggedAddress], - }) - // Still the same single auto-provisioned inbox after the tagged delivery. - expect( - await listEmailInboxesForUser({ db: env.APP_DB, userId }), - ).toHaveLength(1) -}) + const normalizedExistingThread = await createEmailThread({ + db: env.APP_DB, + userId, + inboxId: inbox.id, + subjectNormalized: 'normalized subject', + }) + const subjectOnlyMessage = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw: [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Re: Normalized Subject', + '', + 'Subject-only body.', + ].join('\r\n'), + }) + await handleInboundEmail(subjectOnlyMessage, createInboundEnv()) + const subjectOnly = await listEmailMessages({ + db: env.APP_DB, + userId, + inboxId: inbox.id, + limit: 1, + }) + expect(subjectOnly[0]?.threadId).not.toBe(normalizedExistingThread.id) + + // RFC 5233 subaddressing routes {username}+{tag} to the same inbox and + // keeps the full tagged address in the stored to_addresses so package + // handlers can dispatch on the tag. + const taggedAddress = `${username}+billing@${platformDomain}` + const taggedMessage = createForwardableEmailMessage({ + from: 'invoices@example.net', + to: taggedAddress, + raw: [ + 'From: Invoices ', + `To: ${taggedAddress}`, + 'Subject: Subaddressed mail', + 'Message-ID: ', + '', + 'Tagged body.', + ].join('\r\n'), + }) + await handleInboundEmail(taggedMessage, createInboundEnv()) + expect(taggedMessage.rejectedReason).toBeNull() + const taggedStored = await listEmailMessages({ + db: env.APP_DB, + userId, + inboxId: inbox.id, + limit: 1, + }) + expect(taggedStored[0]).toMatchObject({ + subject: 'Subaddressed mail', + toAddresses: [taggedAddress], + }) + // Still the same single auto-provisioned inbox after the tagged delivery. + expect( + await listEmailInboxesForUser({ db: env.APP_DB, userId }), + ).toHaveLength(1) + }, + inboundRoutingTimeoutMs, +) test('inbound email rejects unknown usernames, reserved locals, and foreign domains', async () => { // Usage recording degrades with a warn when the usage_rollups table is diff --git a/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts index 95451cd0f8..2532aaaf91 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts @@ -13,6 +13,7 @@ import { adminUserGetCapability } from './admin-user-get.ts' import { adminUserListCapability } from './admin-user-list.ts' import { adminUserUpdateCapability } from './admin-user-update.ts' import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' import { loadAdminUserByTarget } from '#worker/admin/users-data.ts' type UserRow = { @@ -504,12 +505,15 @@ function createAdminCapabilityTestDb(input: { return { db, auditEvents, passwordResets, userRoles, users } } +const userMeter = createInMemoryUserMeterEnv() + function createAdminCapabilityContext( db: D1Database, blobs?: Pick, ) { return { env: { + ...userMeter.env, APP_DB: db, EMAIL_BLOBS: blobs ?? { get: async () => null, diff --git a/packages/worker/src/mcp/capabilities/openapi-provider/operation-request.node.test.ts b/packages/worker/src/mcp/capabilities/openapi-provider/operation-request.node.test.ts index 7ebd2a30d5..5f1fd83e70 100644 --- a/packages/worker/src/mcp/capabilities/openapi-provider/operation-request.node.test.ts +++ b/packages/worker/src/mcp/capabilities/openapi-provider/operation-request.node.test.ts @@ -7,8 +7,11 @@ import * as integrationsService from '#worker/integrations/service.ts' import * as packageRepo from '#worker/package-registry/repo.ts' import * as usageModule from '#worker/usage/record-usage.ts' import { type OpenApiBinding } from '#worker/openapi/binding-shared.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' import { executeOpenApiOperationRequest } from './operation-request.ts' +const userMeter = createInMemoryUserMeterEnv() + const spotifyIntegrationConfig = { name: 'spotify', tokenUrl: 'https://accounts.spotify.com/api/token', @@ -137,6 +140,7 @@ function createEnv(entries: Map = new Map()) { } as unknown as D1Database return { + ...userMeter.env, APP_DB: db, SECRET_STORE_KEY: 'test-secret-store-key-32-chars-minimum', } as unknown as Env diff --git a/packages/worker/src/mcp/observability.node.test.ts b/packages/worker/src/mcp/observability.node.test.ts index 3383b2d96c..304f0b63ef 100644 --- a/packages/worker/src/mcp/observability.node.test.ts +++ b/packages/worker/src/mcp/observability.node.test.ts @@ -26,6 +26,10 @@ vi.mock('@sentry/cloudflare', () => ({ sentryMock.captureException(...args), captureMessage: (...args: Array) => sentryMock.captureMessage(...args), + instrumentDurableObjectWithSentry: ( + _getOptions: unknown, + durableObjectClass: unknown, + ) => durableObjectClass, })) const { logMcpEvent } = await import('./observability.ts') diff --git a/packages/worker/src/mcp/tools/execute.node.test.ts b/packages/worker/src/mcp/tools/execute.node.test.ts index bd331a22b1..67df4897a3 100644 --- a/packages/worker/src/mcp/tools/execute.node.test.ts +++ b/packages/worker/src/mcp/tools/execute.node.test.ts @@ -7,6 +7,7 @@ import { } from '#mcp/downstream-mcp-result.ts' import { formatRawFetchHostNudge } from '#mcp/raw-fetch-host-nudge.ts' import type * as RunRecordsServiceModule from '#worker/run-records/service.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' const mockModule = vi.hoisted(() => ({ runModuleWithRegistry: vi.fn(), @@ -59,6 +60,8 @@ vi.mock('#worker/run-records/service.ts', async () => { const { registerExecuteTool } = await import('./execute.ts') +const userMeter = createInMemoryUserMeterEnv() + /** * Minimal env stub: the daily execute entitlement consumed at the top of * the tool handler issues one conditional upsert (allowed when @@ -66,6 +69,7 @@ const { registerExecuteTool } = await import('./execute.ts') * contexts carry no account email (resolves to `max`). */ const stubEnv = { + ...userMeter.env, APP_DB: { prepare() { return { From 73a8e3712c4b337b841b633804585496f60ba58b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 00:25:44 +0000 Subject: [PATCH 06/15] test(entitlements): clean meter test imports Co-authored-by: Kent C. Dodds --- packages/worker/src/entitlements/entitlements.node.test.ts | 1 - packages/worker/src/entitlements/user-meter.workers.test.ts | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/worker/src/entitlements/entitlements.node.test.ts b/packages/worker/src/entitlements/entitlements.node.test.ts index 568a352fdc..024fab0a29 100644 --- a/packages/worker/src/entitlements/entitlements.node.test.ts +++ b/packages/worker/src/entitlements/entitlements.node.test.ts @@ -29,7 +29,6 @@ import { findCachedUserAccountByStableUserId, getCachedUserPlan, getUserPlan, - incrementDailyEntitlementCounter, packageServiceStateStaleMs, refundDailyEntitlement, } from './service.ts' diff --git a/packages/worker/src/entitlements/user-meter.workers.test.ts b/packages/worker/src/entitlements/user-meter.workers.test.ts index 4bfec33bfc..a9f11debc1 100644 --- a/packages/worker/src/entitlements/user-meter.workers.test.ts +++ b/packages/worker/src/entitlements/user-meter.workers.test.ts @@ -11,7 +11,10 @@ import { planLimits } from './plans.ts' import { consumeDailyEntitlement, refundDailyEntitlement } from './service.ts' import { ensureEntitlementTestSchema } from './test-schema.ts' import { userMeterRpc } from './user-meter-client.ts' -import { UserMeter, userMeterMirrorUpdatedAtToken } from './user-meter-do.ts' +import { + type UserMeter, + userMeterMirrorUpdatedAtToken, +} from './user-meter-do.ts' import { createWaitUntilDrain, withPatchedDbPrepare, From 8b001bf8f24850c2fa5d9cc29912e5ece3d1d30c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 00:41:45 +0000 Subject: [PATCH 07/15] style(mcp): format meter gateway tests Co-authored-by: Kent C. Dodds --- .../worker/src/mcp/fetch-gateway.workers.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/worker/src/mcp/fetch-gateway.workers.test.ts b/packages/worker/src/mcp/fetch-gateway.workers.test.ts index 9fa02cee28..dafd062dd9 100644 --- a/packages/worker/src/mcp/fetch-gateway.workers.test.ts +++ b/packages/worker/src/mcp/fetch-gateway.workers.test.ts @@ -71,12 +71,12 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over }) expect(await allowed.text()).toBe('ok') await Promise.all(waitUntilTasks) - expect( - await userMeterRpc({ env, userId }).read({ - resource: 'outbound_fetches_per_day', - day: utcDayKey(), - }), - ).toMatchObject({ outcome: 'ready', count: 1 }) + expect( + await userMeterRpc({ env, userId }).read({ + resource: 'outbound_fetches_per_day', + day: utcDayKey(), + }), + ).toMatchObject({ outcome: 'ready', count: 1 }) // Fill the remaining free-plan quota in UserMeter, then expect denial. await seedOutboundFetchCounter( From df32168fdd3e7a70ded564c67a87849203c7f1e2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 00:53:50 +0000 Subject: [PATCH 08/15] fix(email): read daily usage from UserMeter Co-authored-by: Kent C. Dodds --- packages/worker/src/app/account-email-data.ts | 10 +- .../app/handlers/account-email.node.test.ts | 174 +++++++++++++++--- 2 files changed, 156 insertions(+), 28 deletions(-) diff --git a/packages/worker/src/app/account-email-data.ts b/packages/worker/src/app/account-email-data.ts index 6bacbb4af5..8a6ef3a4ff 100644 --- a/packages/worker/src/app/account-email-data.ts +++ b/packages/worker/src/app/account-email-data.ts @@ -12,8 +12,10 @@ import { } from '#worker/entitlements/plans.ts' import { getUserPlan, + readDailyEntitlementResourceUsage, readEntitlementResourceUsage, } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' import { buildPlatformEmailAddress, getPlatformEmailDomain, @@ -369,6 +371,7 @@ async function countAndListMessages(input: { async function loadUsage(input: { db: D1Database + env: UserMeterEnv userId: string email: string }): Promise { @@ -384,14 +387,16 @@ async function loadUsage(input: { resource: 'stored_email_messages', now, }), - readEntitlementResourceUsage({ + readDailyEntitlementResourceUsage({ db: input.db, + env: input.env, userId: input.userId, resource: 'email_sends_per_day', now, }), - readEntitlementResourceUsage({ + readDailyEntitlementResourceUsage({ db: input.db, + env: input.env, userId: input.userId, resource: 'email_receives_per_day', now, @@ -578,6 +583,7 @@ export async function loadAccountEmailData(input: { loadInboxes({ db: input.env.APP_DB, userId }), loadUsage({ db: input.env.APP_DB, + env: input.env, userId, email: input.user.email, }), diff --git a/packages/worker/src/app/handlers/account-email.node.test.ts b/packages/worker/src/app/handlers/account-email.node.test.ts index d9cc2cdba2..14660553ee 100644 --- a/packages/worker/src/app/handlers/account-email.node.test.ts +++ b/packages/worker/src/app/handlers/account-email.node.test.ts @@ -1,6 +1,9 @@ import { expect, test, vi } from 'vitest' +import { utcDayKey } from '@kody-internal/shared/date-keys.ts' import type * as EmailPlatformAddress from '#worker/email/platform-address.ts' import type * as EntitlementPlans from '#worker/entitlements/plans.ts' +import type * as EntitlementService from '#worker/entitlements/service.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' const messageRow = { id: 'msg-1', @@ -180,11 +183,15 @@ vi.mock('#app/email-verification.ts', () => ({ mockModule.isAccountEmailVerified(...args), })) -vi.mock('#worker/entitlements/service.ts', () => ({ - getUserPlan: (...args: Array) => mockModule.getUserPlan(...args), - readEntitlementResourceUsage: (...args: Array) => - mockModule.readEntitlementResourceUsage(...args), -})) +vi.mock('#worker/entitlements/service.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + getUserPlan: (...args: Array) => mockModule.getUserPlan(...args), + readEntitlementResourceUsage: (...args: Array) => + mockModule.readEntitlementResourceUsage(...args), + } +}) vi.mock('#worker/entitlements/plans.ts', async (importOriginal) => { const actual = await importOriginal() @@ -239,13 +246,47 @@ function createListResult(rows: Array>) { } } -function createEnv() { - let prepareCall = 0 - mockModule.prepare.mockImplementation(() => { - prepareCall += 1 - // countAndListMessages issues COUNT then SELECT in Promise.all order. - if (prepareCall % 2 === 1) return createCountResult(1) - return createListResult([messageRow]) +function createEnv(input?: { + meter?: ReturnType + dailyCounters?: Array<{ + user_id: string + resource: string + day: string + count: number + }> + messageRows?: Array> + messageTotal?: number +}) { + const meter = input?.meter ?? createInMemoryUserMeterEnv() + const dailyCounters = input?.dailyCounters?.map((row) => ({ ...row })) ?? [] + const messageRows = input?.messageRows ?? [messageRow] + const messageTotal = input?.messageTotal ?? messageRows.length + mockModule.prepare.mockImplementation((query: string) => { + const normalized = String(query).replace(/\s+/g, ' ').trim().toLowerCase() + if (normalized.includes('from entitlement_daily_counters')) { + return { + bind(...params: Array) { + return { + async first() { + const row = dailyCounters.find( + (counter) => + counter.user_id === params[0] && + counter.resource === params[1] && + counter.day === params[2], + ) + return (row ? { count: row.count } : { count: 0 }) as T + }, + async all() { + return { results: [] } + }, + } + }, + } + } + if (normalized.includes('count(*)')) { + return createCountResult(messageTotal) + } + return createListResult(messageRows) }) return { APP_DB: { @@ -253,6 +294,7 @@ function createEnv() { } as unknown as D1Database, APP_BASE_URL: 'https://example.com', COOKIE_SECRET: 'secret', + ...meter.env, } as Env } @@ -260,7 +302,26 @@ test('email API lists messages with pagination, usage, and selected detail', asy mockModule.prepare.mockClear() mockModule.getEmailMessageById.mockClear() mockModule.listEmailDeliveryEvents.mockClear() - const env = createEnv() + const day = utcDayKey() + const userId = 'stable-user-1' + const bootstrapMeter = createInMemoryUserMeterEnv() + const env = createEnv({ + meter: bootstrapMeter, + dailyCounters: [ + { + user_id: userId, + resource: 'email_sends_per_day', + day, + count: 2, + }, + { + user_id: userId, + resource: 'email_receives_per_day', + day, + count: 2, + }, + ], + }) const handler = createAccountEmailApiHandler(env) const listResponse = await handler.handler({ @@ -299,6 +360,7 @@ test('email API lists messages with pagination, usage, and selected detail', asy selectedMessage: null, usage: expect.objectContaining({ plan: 'free', + day, stored_messages: { count: 2, limit: 100 }, sends_today: { count: 2, limit: 20 }, receives_today: { count: 2, limit: 50 }, @@ -313,7 +375,22 @@ test('email API lists messages with pagination, usage, and selected detail', asy mockModule.prepare.mockClear() mockModule.getEmailMessageById.mockClear() - const envWithSelection = createEnv() + const envWithSelection = createEnv({ + dailyCounters: [ + { + user_id: userId, + resource: 'email_sends_per_day', + day, + count: 2, + }, + { + user_id: userId, + resource: 'email_receives_per_day', + day, + count: 2, + }, + ], + }) const selectedResponse = await createAccountEmailApiHandler( envWithSelection, ).handler({ @@ -374,6 +451,57 @@ test('email API lists messages with pagination, usage, and selected detail', asy expect(unauthorizedResponse.status).toBe(401) }) +test('email API usage prefers initialized UserMeter daily counts over D1', async () => { + const day = utcDayKey() + const userId = 'stable-user-1' + const meter = createInMemoryUserMeterEnv() + await meter.seed({ + userId, + resource: 'email_sends_per_day', + day, + count: 13, + }) + await meter.seed({ + userId, + resource: 'email_receives_per_day', + day, + count: 15, + }) + const env = createEnv({ + meter, + dailyCounters: [ + { + user_id: userId, + resource: 'email_sends_per_day', + day, + count: 3, + }, + { + user_id: userId, + resource: 'email_receives_per_day', + day, + count: 5, + }, + ], + }) + const response = await createAccountEmailApiHandler(env).handler({ + request: new Request('https://example.com/account/email.json'), + }) + expect(response.status).toBe(200) + const body = await response.json() + expect(body).toMatchObject({ + ok: true, + usage: expect.objectContaining({ + day, + stored_messages: { count: 2, limit: 100 }, + sends_today: { count: 13, limit: 20 }, + receives_today: { count: 15, limit: 50 }, + }), + }) + expect(body.usage.sends_today.count).not.toBe(3) + expect(body.usage.receives_today.count).not.toBe(5) +}) + test('email API lists classification filters and classifies inbound messages', async () => { const quarantinedRow = { ...messageRow, @@ -381,12 +509,10 @@ test('email API lists classification filters and classifies inbound messages', a classification: 'quarantined', classification_reason: 'DMARC failed.', } - const env = createEnv() - let prepareCall = 0 - mockModule.prepare.mockImplementation(() => { - prepareCall += 1 - if (prepareCall % 2 === 1) return createCountResult(1) - return createListResult([quarantinedRow]) + const meter = createInMemoryUserMeterEnv() + const env = createEnv({ + meter, + messageRows: [quarantinedRow], }) mockModule.getEmailMessageById.mockResolvedValueOnce({ ...messageRecord, @@ -420,12 +546,8 @@ test('email API lists classification filters and classifies inbound messages', a }) mockModule.setEmailMessageClassification.mockClear() - let classifyPrepareCall = 0 - mockModule.prepare.mockImplementation(() => { - classifyPrepareCall += 1 - if (classifyPrepareCall % 2 === 1) return createCountResult(1) - return createListResult([messageRow]) - }) + // Reload after classify uses the default message list again. + createEnv({ meter, messageRows: [messageRow] }) const quarantineResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', From e6622a42de6744486dcd6a94cbfb275f87a552e9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 01:15:58 +0000 Subject: [PATCH 09/15] docs(entitlements): clarify meter cutover semantics Co-authored-by: Kent C. Dodds --- .../contributing/architecture/entitlements.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index 5bee0a0a6f..9f8fc15d46 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -144,6 +144,7 @@ dropped in this phase — it remains for existing readers and reporting. with the same cold-bootstrap path) instead of reading D1 directly: - Account usage UI — `packages/worker/src/app/account-usage-data.ts` +- Account email usage panel — `packages/worker/src/app/account-email-data.ts` - `email_usage_get` MCP capability - Admin per-user usage drill-down — `packages/worker/src/admin/user-usage-data.ts` @@ -151,6 +152,12 @@ with the same cold-bootstrap path) instead of reading D1 directly: Non-daily resources and contexts without `USER_METER` still use `readEntitlementResourceUsage` against D1. +During a rolling deployment, requests already running on the previous Worker +version may still increment D1 after a new-version request bootstraps its DO +row. Cloudflare activation bounds that overlap to in-flight requests, but +operators should treat mirror parity during the deploy window as approximate; +post-deploy requests have one authority in UserMeter. + **Inbound retry idempotency:** inbound receive quota uses `UserMeter.consumeInboundDelivery`, which atomically claims `delivery_id` and consumes one `email_receives_per_day` unit inside a SQLite transaction. Retries @@ -324,13 +331,11 @@ Rules: table is not dropped — it remains for existing readers and reporting until reporting-off-D1 retirement is verified. - Only typed pre-commit `RetryableInboundStorageError` failures (thread prework, - R2 put, D1 message/attachment storage after successful cleanup) refund exactly - one `email_receives_per_day` unit via `refundDailyEntitlement` for the same - UTC day that was charged, so Cloudflare Email Routing retries do not burn the - daily receive quota. Post-commit bookkeeping failures do not refund or retry. - Inbound routing retries that replay the same `delivery_id` are idempotent via - `UserMeter.consumeInboundDelivery` and do not consume an additional unit. + A delivery claim remains charged when later storage fails. Cloudflare Email + Routing retries replay that same `delivery_id` through + `UserMeter.consumeInboundDelivery` without incrementing again, including + across a UTC-day boundary. The retained claim is the idempotency boundary; + production inbound handling does not call `refundDailyEntitlement`. `incrementDailyEntitlementCounter` remains for raw D1 counter writes (tests, backfills, and legacy paths). From 7628aaf31365e6142f1698d1f486089b68bb9231 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 01:35:48 +0000 Subject: [PATCH 10/15] fix(entitlements): address UserMeter review feedback Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 8 +- .../contributing/architecture/entitlements.md | 2 +- .../app/handlers/account-email.node.test.ts | 12 +++ .../inbound-entitlements.workers.test.ts | 7 +- .../worker/src/email/inbound.workers.test.ts | 13 ++- .../worker/src/entitlements/user-meter-do.ts | 6 +- .../entitlements/user-meter.workers.test.ts | 83 +++++++++++++++++-- .../email/email-usage-get.workers.test.ts | 13 ++- .../src/mcp/fetch-gateway.workers.test.ts | 17 ++-- 9 files changed, 129 insertions(+), 32 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 25fece6776..f9ef5113b5 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -195,7 +195,7 @@ Durable Object export behavior: section (`exportCounters` RPC; keyset pagination by UTC `day` and `resource`). Retention is self-enforced inside the DO (seven UTC days of counter and inbound-delivery-claim rows); see - [Entitlements](./entitlements.md#user-meter-expand-phase). + [Entitlements](./entitlements.md#usermeter-expand-phase). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -515,7 +515,7 @@ consume/refund/inbound claim, the entitlements service schedules a non-awaited absolute mirror write keyed by `(user_id, resource, day)` with a revision-ordered `updated_at` token (`r/` + zero-padded revision from `userMeterMirrorUpdatedAtToken`) so late writes cannot overwrite newer state. -See [Entitlements](./entitlements.md#user-meter-expand-phase). +See [Entitlements](./entitlements.md#usermeter-expand-phase). **Cold bootstrap:** a missing `(resource, day)` row returns `needs_bootstrap`. The service performs one legacy D1 point read on `entitlement_daily_counters`, @@ -561,7 +561,7 @@ via `durableObjectNameFromParts`); domain helpers such as the DO identity is the user. See [Run records](./run-records.md). - `UserMeter` — `userMeterDurableObjectName(userId)` → `idFromName(userId)`. One daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`). - See [Entitlements](./entitlements.md#user-meter-expand-phase). + See [Entitlements](./entitlements.md#usermeter-expand-phase). - `McpClientHub` — `mcpClientHubDurableObjectName(userId)` → `idFromName(userId.trim())`. - `StorageRunner` — `storageRunnerDurableObjectName(userId, storageId)` → @@ -628,7 +628,7 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` - `RUN_LOG` (Durable Objects; per-user run records — see [Run records](./run-records.md)) - `USER_METER` (Durable Objects; per-user daily entitlement counters — see - [Entitlements](./entitlements.md#user-meter-expand-phase)) + [Entitlements](./entitlements.md#usermeter-expand-phase)) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index 9f8fc15d46..44c202e89b 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -109,7 +109,7 @@ two compute surfaces `usage-metering.md` already observes: Both consume only when the context has a `userId`, matching the usage-metering rule that events without an owning user are skipped. Daily consumption is authoritative in the per-user `UserMeter` Durable Object; see -[UserMeter (expand phase)](#user-meter-expand-phase). +[UserMeter (expand phase)](#usermeter-expand-phase). ## UserMeter (expand phase) diff --git a/packages/worker/src/app/handlers/account-email.node.test.ts b/packages/worker/src/app/handlers/account-email.node.test.ts index 14660553ee..fa111e18b0 100644 --- a/packages/worker/src/app/handlers/account-email.node.test.ts +++ b/packages/worker/src/app/handlers/account-email.node.test.ts @@ -298,10 +298,21 @@ function createEnv(input?: { } as Env } +function useFrozenUtcTime(iso: string) { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(new Date(iso)) + return { + [Symbol.dispose]() { + vi.useRealTimers() + }, + } +} + test('email API lists messages with pagination, usage, and selected detail', async () => { mockModule.prepare.mockClear() mockModule.getEmailMessageById.mockClear() mockModule.listEmailDeliveryEvents.mockClear() + using _frozenTime = useFrozenUtcTime('2026-07-31T15:00:00.000Z') const day = utcDayKey() const userId = 'stable-user-1' const bootstrapMeter = createInMemoryUserMeterEnv() @@ -452,6 +463,7 @@ test('email API lists messages with pagination, usage, and selected detail', asy }) test('email API usage prefers initialized UserMeter daily counts over D1', async () => { + using _frozenTime = useFrozenUtcTime('2026-07-31T15:00:00.000Z') const day = utcDayKey() const userId = 'stable-user-1' const meter = createInMemoryUserMeterEnv() diff --git a/packages/worker/src/email/inbound-entitlements.workers.test.ts b/packages/worker/src/email/inbound-entitlements.workers.test.ts index 41757b39cd..53a9de2f46 100644 --- a/packages/worker/src/email/inbound-entitlements.workers.test.ts +++ b/packages/worker/src/email/inbound-entitlements.workers.test.ts @@ -215,6 +215,7 @@ function withFailingReceiveStartedInsert(input: { const isDeliveryInsert = query.includes('INSERT OR IGNORE INTO email_delivery_events') && query.includes("'receive_started'") + if (!isDeliveryInsert) return statement const originalBind = statement.bind.bind(statement) return { bind(...params: Array) { @@ -224,11 +225,7 @@ function withFailingReceiveStartedInsert(input: { all: bound.all.bind(bound), raw: bound.raw?.bind(bound), run: async () => { - if ( - isDeliveryInsert && - failNext && - params[0] === input.deliveryId - ) { + if (failNext && params[0] === input.deliveryId) { failNext = false throw new Error(input.message) } diff --git a/packages/worker/src/email/inbound.workers.test.ts b/packages/worker/src/email/inbound.workers.test.ts index feadc77b7b..2505321ec9 100644 --- a/packages/worker/src/email/inbound.workers.test.ts +++ b/packages/worker/src/email/inbound.workers.test.ts @@ -864,8 +864,17 @@ async function readUserDailyReceiveCount(userId: string) { } async function readUserTotalReceiveCount(userId: string) { - const exported = await userMeterRpc({ env, userId }).exportCounters({}) - return exported.counters + const meter = userMeterRpc({ env, userId }) + const counters: Array<{ resource: string; count: number }> = [] + let startAfter: string | null = null + for (;;) { + const page = await meter.exportCounters({ startAfter }) + counters.push(...page.counters) + if (!page.truncated) break + if (!page.nextStartAfter) break + startAfter = page.nextStartAfter + } + return counters .filter((row) => row.resource === 'email_receives_per_day') .reduce((total, row) => total + row.count, 0) } diff --git a/packages/worker/src/entitlements/user-meter-do.ts b/packages/worker/src/entitlements/user-meter-do.ts index 36ac9c5213..8616078585 100644 --- a/packages/worker/src/entitlements/user-meter-do.ts +++ b/packages/worker/src/entitlements/user-meter-do.ts @@ -553,8 +553,10 @@ class UserMeterBase extends DurableObject { } async purge(): Promise<{ ok: true }> { - await this.ctx.storage.deleteAll() - this.initializeSchema() + await this.ctx.blockConcurrencyWhile(async () => { + await this.ctx.storage.deleteAll() + this.initializeSchema() + }) return { ok: true } } diff --git a/packages/worker/src/entitlements/user-meter.workers.test.ts b/packages/worker/src/entitlements/user-meter.workers.test.ts index a9f11debc1..d974b3fd63 100644 --- a/packages/worker/src/entitlements/user-meter.workers.test.ts +++ b/packages/worker/src/entitlements/user-meter.workers.test.ts @@ -11,10 +11,7 @@ import { planLimits } from './plans.ts' import { consumeDailyEntitlement, refundDailyEntitlement } from './service.ts' import { ensureEntitlementTestSchema } from './test-schema.ts' import { userMeterRpc } from './user-meter-client.ts' -import { - type UserMeter, - userMeterMirrorUpdatedAtToken, -} from './user-meter-do.ts' +import { UserMeter, userMeterMirrorUpdatedAtToken } from './user-meter-do.ts' import { createWaitUntilDrain, withPatchedDbPrepare, @@ -539,7 +536,20 @@ test('UserMeter daily entitlement consume/refund/read/export/purge workflow is p ]), ) - await meterA.purge() + const [purgeResult, readDuringPurge, exportDuringPurge] = await Promise.all([ + meterA.purge(), + meterA.read({ resource: 'email_sends_per_day', day }).then( + (value) => value, + (error: unknown) => error, + ), + meterA.exportCounters({}).then( + (value) => value, + (error: unknown) => error, + ), + ]) + expect(purgeResult).toEqual({ ok: true }) + expect(readDuringPurge).not.toBeInstanceOf(Error) + expect(exportDuringPurge).not.toBeInstanceOf(Error) expect(await meterA.read({ resource: 'email_sends_per_day', day })).toEqual({ outcome: 'needs_bootstrap', }) @@ -617,3 +627,66 @@ test('UserMeter consume rejects invalid UTC day keys before SQL', async () => { ).rejects.toThrow(/UTC YYYY-MM-DD/) }) }, 30_000) + +test('UserMeter purge blocks concurrent RPCs across deleteAll and schema restore', async () => { + const now = new Date('2026-07-31T15:00:00.000Z') + const day = utcDayKey(now) + const user = await seedFreeUser('meter-purge-concurrency') + const meter = userMeterRpc({ env, userId: user.userId }) + await meter.initialize({ + resource: 'email_sends_per_day', + day, + count: 4, + updatedAt: now.toISOString(), + }) + + const stub = env.USER_METER.get( + env.USER_METER.idFromName(userMeterDurableObjectName(user.userId)), + ) + let releaseDelete: (() => void) | undefined + const deletePaused = new Promise((resolve) => { + releaseDelete = resolve + }) + let deleteAllReached = false + await runInDurableObject(stub, async (instance: UserMeter, state) => { + expect(instance).toBeInstanceOf(UserMeter) + const originalDeleteAll = state.storage.deleteAll.bind(state.storage) + state.storage.deleteAll = async () => { + await originalDeleteAll() + deleteAllReached = true + await deletePaused + } + }) + + const purgePromise = meter.purge() + await waitFor(() => deleteAllReached, 5_000, 'purge deleteAll') + + const readPromise = meter.read({ resource: 'email_sends_per_day', day }).then( + (value) => value, + (error: unknown) => error, + ) + const exportPromise = meter.exportCounters({}).then( + (value) => value, + (error: unknown) => error, + ) + // Give queued RPCs a chance to enter the wiped-schema window if + // blockConcurrencyWhile is missing around deleteAll+initializeSchema. + await new Promise((resolve) => setTimeout(resolve, 25)) + releaseDelete!() + + const [purgeResult, readDuringPurge, exportDuringPurge] = await Promise.all([ + purgePromise, + readPromise, + exportPromise, + ]) + expect(purgeResult).toEqual({ ok: true }) + expect(readDuringPurge).toEqual({ outcome: 'needs_bootstrap' }) + expect(exportDuringPurge).toEqual({ + counters: [], + nextStartAfter: null, + truncated: false, + }) + expect(await meter.read({ resource: 'email_sends_per_day', day })).toEqual({ + outcome: 'needs_bootstrap', + }) +}, 30_000) diff --git a/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts b/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts index 2b71fd5734..4b4b6a211b 100644 --- a/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-usage-get.workers.test.ts @@ -27,6 +27,7 @@ async function seedDailyCounter(input: { userId: string resource: 'email_sends_per_day' | 'email_receives_per_day' count: number + day: string }) { await env.APP_DB.prepare( `INSERT INTO entitlement_daily_counters (user_id, resource, day, count, updated_at) @@ -35,7 +36,7 @@ async function seedDailyCounter(input: { .bind( input.userId, input.resource, - utcDayKey(), + input.day, input.count, new Date().toISOString(), ) @@ -68,6 +69,7 @@ test( { timeout: 30_000 }, async () => { await ensureEmailTestSchema(env.APP_DB) + const day = utcDayKey() await expect( emailUsageGetCapability.handler( {}, @@ -102,11 +104,13 @@ test( userId: planUserId, resource: 'email_sends_per_day', count: 3, + day, }) await seedDailyCounter({ userId: planUserId, resource: 'email_receives_per_day', count: 5, + day, }) await seedStoredMessages(planUserId, 2) @@ -122,7 +126,7 @@ test( ) expect(planResult).toEqual({ plan: 'pro', - day: utcDayKey(), + day, stored_messages: { count: 2, limit: planLimits.pro.maxStoredEmailMessages, @@ -134,7 +138,6 @@ test( }, max_message_bytes: planLimits.pro.maxEmailMessageBytes, }) - const day = utcDayKey() const planMeter = userMeterRpc({ env, userId: planUserId }) expect( await planMeter.read({ resource: 'email_sends_per_day', day }), @@ -158,7 +161,7 @@ test( ) expect(maxResult).toEqual({ plan: 'max', - day: utcDayKey(), + day, stored_messages: { count: 0, limit: maxPlanEmailLimits.stored_email_messages, @@ -181,11 +184,13 @@ test( userId: meterUserId, resource: 'email_sends_per_day', count: 3, + day, }) await seedDailyCounter({ userId: meterUserId, resource: 'email_receives_per_day', count: 5, + day, }) await seedStoredMessages(meterUserId, 2) const updatedAt = new Date().toISOString() diff --git a/packages/worker/src/mcp/fetch-gateway.workers.test.ts b/packages/worker/src/mcp/fetch-gateway.workers.test.ts index dafd062dd9..4c8bbfcecb 100644 --- a/packages/worker/src/mcp/fetch-gateway.workers.test.ts +++ b/packages/worker/src/mcp/fetch-gateway.workers.test.ts @@ -10,6 +10,7 @@ import { UserMeter } from '#worker/entitlements/user-meter-do.ts' import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createWaitUntilDrain } from '#worker/test-support/user-meter.ts' import { seedAccount } from '#worker/test-support/workers-seed.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' @@ -57,20 +58,17 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over } const globalFetch = (async () => new Response('ok')) as unknown as typeof fetch - const waitUntilTasks: Array> = [] - const waitUntil = (promise: Promise) => { - waitUntilTasks.push(promise) - } + const drain = createWaitUntilDrain() const allowed = await executeGatewayFetch({ env, props, request: new Request('https://api.example.net/data'), globalFetch, - waitUntil, + waitUntil: drain.waitUntil, }) expect(await allowed.text()).toBe('ok') - await Promise.all(waitUntilTasks) + await drain.drain() expect( await userMeterRpc({ env, userId }).read({ resource: 'outbound_fetches_per_day', @@ -88,7 +86,7 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over props, request: new Request('https://api.example.net/data'), globalFetch, - waitUntil, + waitUntil: drain.waitUntil, }).catch((error: unknown) => error) expect(isEntitlementLimitError(denied)).toBe(true) @@ -101,7 +99,7 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over props: { ...props, email: null }, request: new Request('https://api.example.net/data'), globalFetch, - waitUntil, + waitUntil: drain.waitUntil, }).catch((error: unknown) => error) expect(isEntitlementLimitError(deniedWithoutEmail)).toBe(true) @@ -112,7 +110,8 @@ test('gateway fetches consume the daily outbound-fetch entitlement and deny over props: { ...props, userId: null, email: null }, request: new Request('https://api.example.net/data'), globalFetch, - waitUntil, + waitUntil: drain.waitUntil, }) expect(await contextless.text()).toBe('ok') + await drain.drain() }, 30_000) From a8cba2b23ca50b89007d1c3ba36230486a02bc31 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 05:43:46 +0000 Subject: [PATCH 11/15] test(entitlements): add RunLog admin fixture Co-authored-by: Kent C. Dodds --- .../src/mcp/capabilities/admin/admin-capabilities.node.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts index 2532aaaf91..7fa3dabbc9 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-capabilities.node.test.ts @@ -12,6 +12,7 @@ import { adminUserCreateCapability } from './admin-user-create.ts' import { adminUserGetCapability } from './admin-user-get.ts' import { adminUserListCapability } from './admin-user-list.ts' import { adminUserUpdateCapability } from './admin-user-update.ts' +import { createInMemoryRunLogUsageEnv } from '#worker/test-support/run-log-usage.ts' import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' import { loadAdminUserByTarget } from '#worker/admin/users-data.ts' @@ -506,6 +507,7 @@ function createAdminCapabilityTestDb(input: { } const userMeter = createInMemoryUserMeterEnv() +const runLog = createInMemoryRunLogUsageEnv() function createAdminCapabilityContext( db: D1Database, @@ -513,6 +515,7 @@ function createAdminCapabilityContext( ) { return { env: { + ...runLog.env, ...userMeter.env, APP_DB: db, EMAIL_BLOBS: blobs ?? { From edfd973adb94436d39c87df42b5b802ae3d4aaf9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 05:57:57 +0000 Subject: [PATCH 12/15] test(run-log): isolate usage counts per user Co-authored-by: Kent C. Dodds --- .../src/admin/user-usage-data.node.test.ts | 101 ++++++++---------- .../src/app/account-usage-data.node.test.ts | 3 +- .../worker/src/test-support/run-log-usage.ts | 14 ++- 3 files changed, 55 insertions(+), 63 deletions(-) diff --git a/packages/worker/src/admin/user-usage-data.node.test.ts b/packages/worker/src/admin/user-usage-data.node.test.ts index a0c549fba0..4cf563112b 100644 --- a/packages/worker/src/admin/user-usage-data.node.test.ts +++ b/packages/worker/src/admin/user-usage-data.node.test.ts @@ -6,14 +6,9 @@ import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { type AdminUsageRollup } from '#app/loader-data.ts' import { loadAdminUserUsageData } from './user-usage-data.ts' -function withUserMeter( - env: { APP_DB: D1Database } & Record, - options?: { activeWorkflowCount?: number }, -) { +function withUserMeter(env: { APP_DB: D1Database } & Record) { const meter = createInMemoryUserMeterEnv() - const runLog = createInMemoryRunLogUsageEnv({ - activeWorkflowCount: options?.activeWorkflowCount ?? 0, - }) + const runLog = createInMemoryRunLogUsageEnv() return { ...env, ...meter.env, ...runLog.env, meter, runLog } } @@ -502,54 +497,52 @@ test('loadAdminUserUsageData reads daily counts from UserMeter (bootstrap then w const meterEmail = 'meter-drilldown@example.com' const meterUserId = await createStableUserIdFromEmail(meterEmail) - const warmEnv = withUserMeter( - { - APP_DB: createAdminUserUsageTestDb({ - users: [ - { - id: 4, - username: 'meter', - email: meterEmail, - plan: 'pro', - stable_user_id: meterUserId, - }, - ], - dailyCounters: [ - { - user_id: meterUserId, - resource: 'email_sends_per_day', - day, - count: 11, - }, - { - user_id: meterUserId, - resource: 'email_receives_per_day', - day, - count: 22, - }, - { - user_id: meterUserId, - resource: 'execute_calls_per_day', - day, - count: 33, - }, - { - user_id: meterUserId, - resource: 'outbound_fetches_per_day', - day, - count: 44, - }, - ], - resourceCounts: { - [meterUserId]: { - saved_packages: 6, - stored_email_messages: 9, - }, + const warmEnv = withUserMeter({ + APP_DB: createAdminUserUsageTestDb({ + users: [ + { + id: 4, + username: 'meter', + email: meterEmail, + plan: 'pro', + stable_user_id: meterUserId, }, - }), - }, - { activeWorkflowCount: 2 }, - ) + ], + dailyCounters: [ + { + user_id: meterUserId, + resource: 'email_sends_per_day', + day, + count: 11, + }, + { + user_id: meterUserId, + resource: 'email_receives_per_day', + day, + count: 22, + }, + { + user_id: meterUserId, + resource: 'execute_calls_per_day', + day, + count: 33, + }, + { + user_id: meterUserId, + resource: 'outbound_fetches_per_day', + day, + count: 44, + }, + ], + resourceCounts: { + [meterUserId]: { + saved_packages: 6, + stored_email_messages: 9, + }, + }, + }), + }) + warmEnv.runLog.setActiveWorkflowCount(meterUserId, 2) await warmEnv.meter.seed({ userId: meterUserId, resource: 'email_sends_per_day', diff --git a/packages/worker/src/app/account-usage-data.node.test.ts b/packages/worker/src/app/account-usage-data.node.test.ts index a26c0b5e5d..e672a0c141 100644 --- a/packages/worker/src/app/account-usage-data.node.test.ts +++ b/packages/worker/src/app/account-usage-data.node.test.ts @@ -178,7 +178,8 @@ test('loadAccountUsageData returns plan rows and authoritative UserMeter daily c ], }) const warmEnv = withUsageEnv({ APP_DB: warmDb }) - warmEnv.runLog.setActiveWorkflowCount(3) + warmEnv.runLog.setActiveWorkflowCount(meterUserId, 3) + warmEnv.runLog.setActiveWorkflowCount('other-user', 99) await warmEnv.meter.seed({ userId: meterUserId, resource: 'email_sends_per_day', diff --git a/packages/worker/src/test-support/run-log-usage.ts b/packages/worker/src/test-support/run-log-usage.ts index 39876fce38..2742a17b64 100644 --- a/packages/worker/src/test-support/run-log-usage.ts +++ b/packages/worker/src/test-support/run-log-usage.ts @@ -2,23 +2,21 @@ * Minimal in-memory RUN_LOG stub for node-unit usage-reader coverage of * concurrent_workflows (authoritative countActiveWorkflowProjections). */ -export function createInMemoryRunLogUsageEnv(input?: { - activeWorkflowCount?: number -}) { - let activeWorkflowCount = input?.activeWorkflowCount ?? 0 +export function createInMemoryRunLogUsageEnv() { + const activeWorkflowCounts = new Map() return { env: { RUN_LOG: { idFromName: (name: string) => name as unknown as DurableObjectId, - get: () => ({ + get: (id: DurableObjectId) => ({ countActiveWorkflowProjections: async () => ({ - count: activeWorkflowCount, + count: activeWorkflowCounts.get(String(id)) ?? 0, }), }), }, }, - setActiveWorkflowCount(count: number) { - activeWorkflowCount = Math.max(0, Math.trunc(count) || 0) + setActiveWorkflowCount(id: string, count: number) { + activeWorkflowCounts.set(id, Math.max(0, Math.trunc(count) || 0)) }, } } From b18246f79e66c0172e6b9f07a327ddb4288cdf2b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 06:56:32 +0000 Subject: [PATCH 13/15] feat(entitlements): shadow storage bytes in UserMeter Co-authored-by: Kent C. Dodds --- .../entitlements/d1-storage-reconciliation.ts | 6 + .../d1-storage-reconciliation.workers.test.ts | 114 ++++++- packages/worker/src/entitlements/service.ts | 88 ++++- .../worker/src/entitlements/user-meter-do.ts | 245 +++++++++++++- .../entitlements/user-meter.workers.test.ts | 308 +++++++++++++++++- packages/worker/src/mcp/memory/service.ts | 7 +- packages/worker/src/mcp/secrets/service.ts | 27 +- .../src/mcp/values/service.node.test.ts | 126 +++++++ packages/worker/src/mcp/values/service.ts | 6 +- .../src/package-registry/service.node.test.ts | 3 + .../worker/src/package-registry/service.ts | 3 + .../storage-runner.entitlement.node.test.ts | 49 +++ .../worker/src/storage-runner.workers.test.ts | 1 + .../worker/src/test-support/user-meter.ts | 100 +++++- 14 files changed, 1068 insertions(+), 15 deletions(-) diff --git a/packages/worker/src/entitlements/d1-storage-reconciliation.ts b/packages/worker/src/entitlements/d1-storage-reconciliation.ts index 7428ed8496..0759de7b6c 100644 --- a/packages/worker/src/entitlements/d1-storage-reconciliation.ts +++ b/packages/worker/src/entitlements/d1-storage-reconciliation.ts @@ -1,3 +1,4 @@ +import { type UserMeterEnv } from './user-meter-client.ts' import { listUsersForD1StorageReconciliation, markUserD1StorageReconciliationAttempt, @@ -9,6 +10,8 @@ import { * Reservations keep the hot path conservative between sweeps; this lane * corrects drift from deletes, failed writes after reservation, and D1 * surfaces whose writes do not pass through storage-byte enforcement. + * Optional `env` best-effort shadows each absolute into UserMeter after the + * D1 update; shadow failures never fail or poison the lane. */ export const d1StorageReconciliationBatchSize = 8 @@ -16,6 +19,8 @@ export async function reconcileD1StorageBytes(input: { db: D1Database now?: Date batchSize?: number + /** Optional expand-phase UserMeter shadow after each D1 absolute update. */ + env?: UserMeterEnv }): Promise<{ scanned: number; updated: number; failed: number }> { const now = input.now ?? new Date() const rows = await listUsersForD1StorageReconciliation({ @@ -28,6 +33,7 @@ export async function reconcileD1StorageBytes(input: { try { const result = await reconcileUserD1StorageBytes({ db: input.db, + env: input.env, userId: row.userId, now, }) diff --git a/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts b/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts index d9daeacd68..a690d38491 100644 --- a/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts +++ b/packages/worker/src/entitlements/d1-storage-reconciliation.workers.test.ts @@ -1,11 +1,13 @@ import { env } from 'cloudflare:test' import { expect, test } from 'vitest' import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' import { calculateUserD1StorageBytes, readUserD1StorageBytes, } from './service.ts' import { reconcileD1StorageBytes } from './d1-storage-reconciliation.ts' +import { userMeterRpc } from './user-meter-client.ts' async function insertReconciliationUser(input: { userId: string @@ -29,7 +31,7 @@ async function insertReconciliationUser(input: { .run() } -test('D1 storage reconciliation is bounded and replaces stored drift with authoritative bytes', async () => { +test('D1 storage reconciliation is bounded and remains D1-authoritative with {db}', async () => { await ensureEmailTestSchema(env.APP_DB) await env.APP_DB.prepare( `UPDATE users @@ -71,6 +73,11 @@ test('D1 storage reconciliation is bounded and replaces stored drift with author }) expect(expectedFirstBytes).toBeGreaterThan(128) + const firstMeter = userMeterRpc({ env, userId: firstUserId }) + expect(await firstMeter.readStorageBytes()).toEqual({ + outcome: 'needs_bootstrap', + }) + await expect( reconcileD1StorageBytes({ db: env.APP_DB, @@ -78,9 +85,13 @@ test('D1 storage reconciliation is bounded and replaces stored drift with author batchSize: 1, }), ).resolves.toEqual({ scanned: 1, updated: 1, failed: 0 }) + await expect( readUserD1StorageBytes({ db: env.APP_DB, userId: firstUserId }), ).resolves.toBe(expectedFirstBytes) + expect(await firstMeter.readStorageBytes()).toEqual({ + outcome: 'needs_bootstrap', + }) await expect( readUserD1StorageBytes({ db: env.APP_DB, userId: secondUserId }), ).resolves.toBe(99) @@ -96,3 +107,104 @@ test('D1 storage reconciliation is bounded and replaces stored drift with author readUserD1StorageBytes({ db: env.APP_DB, userId: secondUserId }), ).resolves.toBe(0) }) + +test('optional reconciliation env shadows UserMeter without becoming authoritative', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = '3'.repeat(64) + await insertReconciliationUser({ + userId, + email: `${crypto.randomUUID()}@example.test`, + storedBytes: 50, + updatedAt: '2019-01-01T00:00:00.000Z', + }) + await env.APP_DB.prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, text_body, raw_size, + processing_status, created_at, updated_at + ) VALUES (?, 'inbound', ?, 'sender@example.test', 'subject', 'body', 200, + 'stored', ?, ?)`, + ) + .bind( + crypto.randomUUID(), + userId, + '2026-07-31T00:00:00.000Z', + '2026-07-31T00:00:00.000Z', + ) + .run() + + const meter = userMeterRpc({ env, userId }) + await meter.setStorageBytes({ + bytes: 999_999, + updatedAt: '2026-07-31T00:00:00.000Z', + }) + + const expectedBytes = await calculateUserD1StorageBytes({ + db: env.APP_DB, + userId, + }) + await expect( + reconcileD1StorageBytes({ + db: env.APP_DB, + env, + now: new Date('2026-07-31T01:00:00.000Z'), + batchSize: 8, + }), + ).resolves.toMatchObject({ failed: 0 }) + + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId }), + ).resolves.toBe(expectedBytes) + expect(await meter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: expectedBytes, + }) +}) + +test('reconciliation shadow failure does not fail or poison the D1 lane', async () => { + await ensureEmailTestSchema(env.APP_DB) + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes_updated_at = '9999-12-31T23:59:59.999Z'`, + ).run() + const userId = '4'.repeat(64) + await insertReconciliationUser({ + userId, + email: `${crypto.randomUUID()}@example.test`, + storedBytes: 40, + updatedAt: '2018-01-01T00:00:00.000Z', + }) + + consoleWarn.mockImplementation(() => {}) + const failingMeterEnv = { + USER_METER: { + idFromName: env.USER_METER.idFromName.bind(env.USER_METER), + get() { + return { + async setStorageBytes() { + throw new Error('shadow write failed') + }, + } + }, + }, + } as unknown as typeof env + + const expectedBytes = await calculateUserD1StorageBytes({ + db: env.APP_DB, + userId, + }) + await expect( + reconcileD1StorageBytes({ + db: env.APP_DB, + env: failingMeterEnv, + now: new Date('2026-07-31T01:00:00.000Z'), + batchSize: 1, + }), + ).resolves.toEqual({ scanned: 1, updated: 1, failed: 0 }) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId }), + ).resolves.toBe(expectedBytes) + expect(consoleWarn).toHaveBeenCalledWith( + 'entitlement-storage-bytes-shadow-failed', + expect.any(Error), + ) +}) diff --git a/packages/worker/src/entitlements/service.ts b/packages/worker/src/entitlements/service.ts index 93d8e72435..ac3d13183b 100644 --- a/packages/worker/src/entitlements/service.ts +++ b/packages/worker/src/entitlements/service.ts @@ -13,7 +13,11 @@ import { isDailyEntitlementResource, type DailyEntitlementResource, } from './user-meter-do.ts' -import { userMeterRpc, type UserMeterEnv } from './user-meter-client.ts' +import { + userMeterNamespace, + userMeterRpc, + type UserMeterEnv, +} from './user-meter-client.ts' /** Env surface for authoritative entitlement usage readers (UserMeter + RunLog). */ export type EntitlementUsageEnv = UserMeterEnv & Pick @@ -682,10 +686,49 @@ export async function calculateUserD1StorageBytes(input: { return sums.reduce((total, value) => total + value, 0) } -export async function readUserD1StorageBytes(input: { +/** Best-effort D1→UserMeter absolute shadow; re-reads D1 at task time. */ +async function shadowUserMeterStorageBytesFromD1(input: { db: D1Database + env: UserMeterEnv userId: string }) { + if (!userMeterNamespace(input.env)) return + const bytes = await readUserD1StorageBytes({ + db: input.db, + userId: input.userId, + }) + const meter = userMeterRpc({ env: input.env, userId: input.userId }) + await meter.setStorageBytes({ + bytes, + updatedAt: new Date().toISOString(), + }) +} + +function scheduleUserMeterStorageBytesShadow(input: { + db: D1Database + env: UserMeterEnv | undefined + userId: string + waitUntil?: (promise: Promise) => void +}) { + if (!input.env || !userMeterNamespace(input.env)) return + const tracked = shadowUserMeterStorageBytesFromD1({ + db: input.db, + env: input.env, + userId: input.userId, + }).catch((error: unknown) => { + console.warn('entitlement-storage-bytes-shadow-failed', error) + }) + if (input.waitUntil) { + input.waitUntil(tracked) + return + } + void tracked +} + +export async function readUserD1StorageBytes(input: { + db: D1Database + userId: string +}): Promise { const row = await input.db .prepare( `SELECT d1_storage_bytes AS bytes @@ -697,10 +740,17 @@ export async function readUserD1StorageBytes(input: { return Math.max(0, Number(row?.bytes ?? 0)) } +/** + * Replace one user's D1 storage-byte counter with the cross-surface D1 payload + * sum. Optional `env` best-effort shadows into UserMeter after the D1 write; + * shadow failures never fail or poison this reconcile. + */ export async function reconcileUserD1StorageBytes(input: { db: D1Database userId: string now?: Date + /** Optional expand-phase UserMeter shadow after the D1 absolute update. */ + env?: UserMeterEnv }) { const bytes = await calculateUserD1StorageBytes(input) const result = await input.db @@ -712,7 +762,19 @@ export async function reconcileUserD1StorageBytes(input: { ) .bind(bytes, (input.now ?? new Date()).toISOString(), input.userId) .run() - return { bytes, updated: (result.meta.changes ?? 0) > 0 } + const updated = (result.meta.changes ?? 0) > 0 + if (input.env) { + try { + await shadowUserMeterStorageBytesFromD1({ + db: input.db, + env: input.env, + userId: input.userId, + }) + } catch (error) { + console.warn('entitlement-storage-bytes-shadow-failed', error) + } + } + return { bytes, updated } } export async function listUsersForD1StorageReconciliation(input: { @@ -923,12 +985,22 @@ export async function readCurrentEntitlementResourceUsage(input: { }) } +/** + * Atomically reserve D1 payload storage bytes against the plan limit. + * Optional `env` schedules a best-effort UserMeter shadow after a successful + * reserve (never awaited; failures logged only). `getCurrent` is check-only + * and does not reserve in D1 or UserMeter. + */ export async function assertWithinStorageBytesEntitlement(input: { db: D1Database userId: string email: string | null | undefined requested?: number getCurrent?: () => Promise + /** Optional expand-phase UserMeter shadow after a successful D1 reserve. */ + env?: UserMeterEnv + /** Prefer `ctx.waitUntil` for the UserMeter shadow; never awaited here. */ + waitUntil?: (promise: Promise) => void }) { if (input.getCurrent) { await assertWithinEntitlement({ @@ -959,7 +1031,15 @@ export async function assertWithinStorageBytesEntitlement(input: { ) .bind(requested, now, input.userId, requested, limit) .run() - if ((result.meta.changes ?? 0) > 0) return + if ((result.meta.changes ?? 0) > 0) { + scheduleUserMeterStorageBytesShadow({ + db: input.db, + env: input.env, + userId: input.userId, + waitUntil: input.waitUntil, + }) + return + } const current = await readUserD1StorageBytes({ db: input.db, diff --git a/packages/worker/src/entitlements/user-meter-do.ts b/packages/worker/src/entitlements/user-meter-do.ts index 8616078585..adc12de48d 100644 --- a/packages/worker/src/entitlements/user-meter-do.ts +++ b/packages/worker/src/entitlements/user-meter-do.ts @@ -26,7 +26,9 @@ export const userMeterDailyCounterRetentionDays = 7 const metaSchemaVersionKey = 'schema_version' /** Bump when initializeSchema DDL changes; warm objects skip DDL. */ -const userMeterSchemaVersion = 3 +const userMeterSchemaVersion = 4 +/** Singleton row id for expand-phase D1 storage-byte shadow (schema v4). */ +const storageBytesStateRowId = 1 const defaultExportPageSize = 100 const maxExportPageSize = 500 @@ -77,8 +79,41 @@ export type UserMeterInitializeResult = UserMeterReadyState & { created: boolean } +export type UserMeterStorageBytesState = { + bytes: number + revision: number + updatedAt: string + mirrorUpdatedAt: string +} + +export type UserMeterStorageBytesReadyState = { + outcome: 'ready' + bytes: number + revision: number + mirrorUpdatedAt: string +} + +export type UserMeterStorageBytesReadResult = + | UserMeterBootstrapState + | UserMeterStorageBytesReadyState + +export type UserMeterStorageBytesReserveResult = + | UserMeterBootstrapState + | (UserMeterStorageBytesReadyState & { reserved: boolean }) + +export type UserMeterStorageBytesInitializeResult = + UserMeterStorageBytesReadyState & { + created: boolean + } + +export type UserMeterStorageBytesSetResult = UserMeterStorageBytesReadyState & { + created: boolean +} + export type UserMeterExportResult = { counters: Array + /** Non-authoritative D1 storage-byte shadow; `null` when never synced. */ + storageBytesShadow: UserMeterStorageBytesState | null nextStartAfter: string | null truncated: boolean } @@ -253,6 +288,15 @@ class UserMeterBase extends DurableObject { `CREATE INDEX IF NOT EXISTS idx_inbound_delivery_claims_day ON inbound_delivery_claims (day)`, ) + // Expand-phase shadow of D1 `users.d1_storage_bytes` (schema v4). + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS storage_bytes_state ( + id INTEGER PRIMARY KEY NOT NULL CHECK (id = 1), + bytes INTEGER NOT NULL, + revision INTEGER NOT NULL, + updated_at TEXT NOT NULL + ) + `) this.ctx.storage.sql.exec( `INSERT INTO user_meter_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, @@ -261,6 +305,41 @@ class UserMeterBase extends DurableObject { ) } + private storageReadyState( + bytes: number, + revision: number, + ): UserMeterStorageBytesReadyState { + const safeBytes = Math.max(0, bytes) + const safeRevision = Math.max(0, revision) + return { + outcome: 'ready', + bytes: safeBytes, + revision: safeRevision, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(safeRevision), + } + } + + private readStorageRow(): { + bytes: number + revision: number + updatedAt: string + } | null { + const row = this.ctx.storage.sql + .exec<{ bytes: number; revision: number; updated_at: string }>( + `SELECT bytes, revision, updated_at + FROM storage_bytes_state + WHERE id = ?`, + storageBytesStateRowId, + ) + .toArray()[0] + if (!row) return null + return { + bytes: Math.max(0, Number(row.bytes ?? 0)), + revision: Math.max(0, Number(row.revision ?? 0)), + updatedAt: String(row.updated_at ?? ''), + } + } + private deleteStaleCounters(now: Date) { const cutoffDay = retentionCutoffDay(now) this.ctx.storage.sql.exec( @@ -552,6 +631,143 @@ class UserMeterBase extends DurableObject { return readyState(row.count, row.revision) } + /** Cutover-support cold seed; expand-phase shadow sync uses {@link setStorageBytes}. */ + async initializeStorageBytes(input: { + bytes: number + updatedAt: string + }): Promise { + const bytes = Math.max(0, Math.trunc(Number(input.bytes) || 0)) + const cursor = this.ctx.storage.sql.exec( + `INSERT INTO storage_bytes_state (id, bytes, revision, updated_at) + VALUES (?, ?, 1, ?) + ON CONFLICT(id) DO NOTHING`, + storageBytesStateRowId, + bytes, + input.updatedAt, + ) + const created = cursor.rowsWritten > 0 + const row = this.readStorageRow() + if (!row) { + throw new Error( + 'UserMeter initializeStorageBytes failed to materialize storage state.', + ) + } + return { ...this.storageReadyState(row.bytes, row.revision), created } + } + + /** Cutover-support / shadow read; expand-phase usage and enforcement use D1. */ + async readStorageBytes(): Promise { + const row = this.readStorageRow() + if (!row) return { outcome: 'needs_bootstrap' } + return this.storageReadyState(row.bytes, row.revision) + } + + /** + * Cutover-support atomic reserve. Expand-phase enforcement uses D1 only; + * missing state returns `needs_bootstrap`. + */ + async reserveStorageBytes(input: { + requested: number + limit: number + updatedAt: string + }): Promise { + const requested = Math.max(0, Math.trunc(Number(input.requested) || 0)) + const existing = this.readStorageRow() + if (!existing) { + return { outcome: 'needs_bootstrap' } + } + if ( + requested > 0 && + (input.limit < 1 || existing.bytes + requested > input.limit) + ) { + return { + ...this.storageReadyState(existing.bytes, existing.revision), + reserved: false, + } + } + if (requested === 0) { + return { + ...this.storageReadyState(existing.bytes, existing.revision), + reserved: true, + } + } + const nextBytes = existing.bytes + requested + const nextRevision = existing.revision + 1 + this.ctx.storage.sql.exec( + `UPDATE storage_bytes_state + SET bytes = ?, + revision = ?, + updated_at = ? + WHERE id = ? AND revision = ?`, + nextBytes, + nextRevision, + input.updatedAt, + storageBytesStateRowId, + existing.revision, + ) + const row = this.readStorageRow() + if (!row) { + throw new Error('UserMeter reserveStorageBytes lost storage state.') + } + return { + ...this.storageReadyState(row.bytes, row.revision), + reserved: row.bytes === nextBytes && row.revision === nextRevision, + } + } + + /** + * Absolute shadow-set from D1. Materializes the singleton and bumps + * revision. Callers should re-read latest D1 before invoking so delayed + * shadows cannot leave the DO behind D1. + */ + async setStorageBytes(input: { + bytes: number + updatedAt: string + }): Promise { + const bytes = Math.max(0, Math.trunc(Number(input.bytes) || 0)) + const existing = this.readStorageRow() + if (!existing) { + this.ctx.storage.sql.exec( + `INSERT INTO storage_bytes_state (id, bytes, revision, updated_at) + VALUES (?, ?, 1, ?)`, + storageBytesStateRowId, + bytes, + input.updatedAt, + ) + const row = this.readStorageRow() + if (!row) { + throw new Error( + 'UserMeter setStorageBytes failed to materialize storage state.', + ) + } + return { + ...this.storageReadyState(row.bytes, row.revision), + created: true, + } + } + const nextRevision = existing.revision + 1 + this.ctx.storage.sql.exec( + `UPDATE storage_bytes_state + SET bytes = ?, + revision = ?, + updated_at = ? + WHERE id = ? AND revision = ?`, + bytes, + nextRevision, + input.updatedAt, + storageBytesStateRowId, + existing.revision, + ) + const row = this.readStorageRow() + if (!row) { + throw new Error('UserMeter setStorageBytes lost storage state.') + } + return { + ...this.storageReadyState(row.bytes, row.revision), + created: false, + } + } + async purge(): Promise<{ ok: true }> { await this.ctx.blockConcurrencyWhile(async () => { await this.ctx.storage.deleteAll() @@ -622,9 +838,18 @@ class UserMeterBase extends DurableObject { mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(revision), }) } + const storageRow = this.readStorageRow() const last = pageRows[pageRows.length - 1] return { counters, + storageBytesShadow: storageRow + ? { + bytes: storageRow.bytes, + revision: storageRow.revision, + updatedAt: storageRow.updatedAt, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(storageRow.revision), + } + : null, nextStartAfter: truncated && last ? encodeExportCursor({ @@ -672,6 +897,24 @@ export type UserMeterRpc = { day: string updatedAt: string }) => Promise + /** Cutover-support cold seed; expand-phase shadow uses setStorageBytes. */ + initializeStorageBytes: (input: { + bytes: number + updatedAt: string + }) => Promise + /** Cutover-support / shadow read; expand-phase usage reads D1. */ + readStorageBytes: () => Promise + /** Cutover-support reserve; expand-phase enforcement uses D1. */ + reserveStorageBytes: (input: { + requested: number + limit: number + updatedAt: string + }) => Promise + /** Expand-phase absolute shadow-set from D1. */ + setStorageBytes: (input: { + bytes: number + updatedAt: string + }) => Promise purge: () => Promise<{ ok: true }> exportCounters: (input: { pageSize?: number diff --git a/packages/worker/src/entitlements/user-meter.workers.test.ts b/packages/worker/src/entitlements/user-meter.workers.test.ts index d974b3fd63..3722d4b156 100644 --- a/packages/worker/src/entitlements/user-meter.workers.test.ts +++ b/packages/worker/src/entitlements/user-meter.workers.test.ts @@ -8,7 +8,12 @@ import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { userMeterDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' import { EntitlementLimitError } from './errors.ts' import { planLimits } from './plans.ts' -import { consumeDailyEntitlement, refundDailyEntitlement } from './service.ts' +import { + assertWithinStorageBytesEntitlement, + consumeDailyEntitlement, + readUserD1StorageBytes, + refundDailyEntitlement, +} from './service.ts' import { ensureEntitlementTestSchema } from './test-schema.ts' import { userMeterRpc } from './user-meter-client.ts' import { UserMeter, userMeterMirrorUpdatedAtToken } from './user-meter-do.ts' @@ -555,6 +560,7 @@ test('UserMeter daily entitlement consume/refund/read/export/purge workflow is p }) expect(await meterA.exportCounters({})).toEqual({ counters: [], + storageBytesShadow: null, nextStartAfter: null, truncated: false, }) @@ -683,6 +689,7 @@ test('UserMeter purge blocks concurrent RPCs across deleteAll and schema restore expect(readDuringPurge).toEqual({ outcome: 'needs_bootstrap' }) expect(exportDuringPurge).toEqual({ counters: [], + storageBytesShadow: null, nextStartAfter: null, truncated: false, }) @@ -690,3 +697,302 @@ test('UserMeter purge blocks concurrent RPCs across deleteAll and schema restore outcome: 'needs_bootstrap', }) }, 30_000) + +test('storage bytes enforcement remains D1-authoritative even when env is passed', async () => { + const user = await seedFreeUser('meter-storage-d1-authority') + const drain = createWaitUntilDrain() + const meter = userMeterRpc({ env, userId: user.userId }) + const storageLimit = planLimits.free.maxStorageBytes + const legacyBytes = storageLimit - 10 + + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(legacyBytes, '2026-07-31T12:00:00.000Z', user.userId) + .run() + + await meter.setStorageBytes({ + bytes: storageLimit, + updatedAt: '2026-07-31T11:00:00.000Z', + }) + + await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + requested: 5, + waitUntil: drain.waitUntil, + }) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(legacyBytes + 5) + + const denied = await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + requested: 6, + waitUntil: drain.waitUntil, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(denied).toBeInstanceOf(EntitlementLimitError) + expect(denied).toMatchObject({ + details: { + resource: 'storage_bytes', + plan: 'free', + limit: storageLimit, + current: legacyBytes + 5, + }, + }) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(legacyBytes + 5) + + await drain.drain() + expect(await meter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: legacyBytes + 5, + }) +}, 30_000) + +test('UserMeter storage shadow failure cannot affect successful D1 reserves', async () => { + const user = await seedFreeUser('meter-storage-shadow-fail') + const drain = createWaitUntilDrain() + const storageLimit = planLimits.free.maxStorageBytes + + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(storageLimit - 20, '2026-07-31T12:00:00.000Z', user.userId) + .run() + + consoleWarn.mockImplementation(() => {}) + const failingMeterEnv = { + USER_METER: { + idFromName: env.USER_METER.idFromName.bind(env.USER_METER), + get() { + return { + async setStorageBytes() { + throw new Error('shadow write failed') + }, + } + }, + }, + } as unknown as typeof env + + await expect( + assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env: failingMeterEnv, + userId: user.userId, + email: user.email, + requested: 5, + waitUntil: drain.waitUntil, + }), + ).resolves.toBeUndefined() + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(storageLimit - 15) + await drain.drain() + expect(consoleWarn).toHaveBeenCalledWith( + 'entitlement-storage-bytes-shadow-failed', + expect.any(Error), + ) +}, 30_000) + +test('concurrent D1 storage reserves cannot overshoot the plan limit', async () => { + const user = await seedFreeUser('meter-storage-concurrent-d1') + const limit = planLimits.free.maxStorageBytes + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(limit - 10, '2026-07-31T15:00:00.000Z', user.userId) + .run() + + const attempts = await Promise.all( + Array.from({ length: 20 }, () => + assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + userId: user.userId, + email: user.email, + requested: 5, + }).then( + () => 'reserved' as const, + (error: unknown) => error, + ), + ), + ) + const reserved = attempts.filter((result) => result === 'reserved') + const denied = attempts.filter( + (result) => result instanceof EntitlementLimitError, + ) + expect(reserved).toHaveLength(2) + expect(denied).toHaveLength(18) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(limit) +}, 30_000) + +test('delayed UserMeter storage shadow re-reads latest D1 and cannot regress D1', async () => { + const user = await seedFreeUser('meter-storage-shadow-latest') + const drain = createWaitUntilDrain() + const meter = userMeterRpc({ env, userId: user.userId }) + + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(10, '2026-07-31T16:00:00.000Z', user.userId) + .run() + + let releaseShadow!: () => void + const shadowGate = new Promise((resolve) => { + releaseShadow = resolve + }) + let shadowReads = 0 + using _patch = withPatchedDbPrepare(env.APP_DB, (originalPrepare) => { + return ((query: string) => { + const statement = originalPrepare(query) + const isPointRead = + query.includes('SELECT d1_storage_bytes AS bytes') && + query.includes('FROM users') + const isReserve = query.includes( + 'SET d1_storage_bytes = d1_storage_bytes + ?', + ) + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: async () => { + if (isPointRead) { + shadowReads += 1 + await shadowGate + } + return await bound.first() + }, + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: bound.run.bind(bound), + } + }, + } + }) as D1Database['prepare'] + }) + + await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: user.userId, + email: user.email, + requested: 5, + waitUntil: drain.waitUntil, + }) + await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + userId: user.userId, + email: user.email, + requested: 7, + }) + releaseShadow() + await drain.drain() + expect(shadowReads).toBeGreaterThanOrEqual(1) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(22) + expect(await meter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: 22, + }) +}, 30_000) + +test('UserMeter storage RPCs, export shadow field, and purge work additively', async () => { + const user = await seedFreeUser('meter-storage-export-purge') + const meter = userMeterRpc({ env, userId: user.userId }) + await meter.initializeStorageBytes({ + bytes: 42, + updatedAt: '2026-07-31T17:00:00.000Z', + }) + await meter.reserveStorageBytes({ + requested: 8, + limit: 1_000, + updatedAt: '2026-07-31T17:01:00.000Z', + }) + await meter.setStorageBytes({ + bytes: 11, + updatedAt: '2026-07-31T17:02:00.000Z', + }) + + const exported = await meter.exportCounters({}) + expect(exported.storageBytesShadow).toEqual({ + bytes: 11, + revision: 3, + updatedAt: '2026-07-31T17:02:00.000Z', + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(3), + }) + await expect( + readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), + ).resolves.toBe(0) + + await expect(meter.purge()).resolves.toEqual({ ok: true }) + expect(await meter.readStorageBytes()).toEqual({ + outcome: 'needs_bootstrap', + }) + expect(await meter.exportCounters({})).toEqual({ + counters: [], + storageBytesShadow: null, + nextStartAfter: null, + truncated: false, + }) +}, 30_000) + +test('missing-user storage reserve preserves EntitlementLimitError semantics on D1 path', async () => { + await ensureEntitlementTestSchema(env.APP_DB) + const missingUserId = 'a'.repeat(64) + const freeLimit = planLimits.free.maxStorageBytes + + await expect( + assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: missingUserId, + email: null, + requested: 1, + }), + ).resolves.toBeUndefined() + + const denied = await assertWithinStorageBytesEntitlement({ + db: env.APP_DB, + env, + userId: missingUserId, + email: null, + requested: freeLimit + 1, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(denied).toBeInstanceOf(EntitlementLimitError) + expect(denied).toMatchObject({ + details: { + resource: 'storage_bytes', + plan: 'free', + limit: freeLimit, + current: 0, + }, + }) +}, 30_000) diff --git a/packages/worker/src/mcp/memory/service.ts b/packages/worker/src/mcp/memory/service.ts index 993b24b0c9..846afba466 100644 --- a/packages/worker/src/mcp/memory/service.ts +++ b/packages/worker/src/mcp/memory/service.ts @@ -21,6 +21,7 @@ import { assertWithinStorageBytesEntitlement, estimateEntitlementStorageEntryByteDelta, } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' import { type McpMemoryRow, type MemoryRecord, @@ -72,7 +73,8 @@ type MemoryOwnerContext = { } type MemoryEnv = Pick & - Partial> + Partial> & + UserMeterEnv type MemoryUpsertInput = MemoryOwnerContext & { env: MemoryEnv @@ -86,6 +88,7 @@ type MemoryUpsertInput = MemoryOwnerContext & { dedupeKey?: string | null status?: 'active' | 'archived' verificationReference?: string | null + waitUntil?: (promise: Promise) => void } type MemoryDeleteInput = MemoryOwnerContext & { @@ -215,6 +218,7 @@ export async function upsertMemory(input: MemoryUpsertInput): Promise<{ await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -247,6 +251,7 @@ export async function upsertMemory(input: MemoryUpsertInput): Promise<{ } : null, }), + waitUntil: input.waitUntil, }) if (existing) { diff --git a/packages/worker/src/mcp/secrets/service.ts b/packages/worker/src/mcp/secrets/service.ts index e57cd35455..8440cf7ed1 100644 --- a/packages/worker/src/mcp/secrets/service.ts +++ b/packages/worker/src/mcp/secrets/service.ts @@ -39,6 +39,7 @@ import { assertWithinStorageBytesEntitlement, estimateEntitlementStorageEntryByteDelta, } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' import { type SecretMetadata, type SecretScope } from './types.ts' type SecretOwnerContext = { @@ -46,14 +47,17 @@ type SecretOwnerContext = { storageContext?: StorageContext | null } +type SecretWriteEnv = Pick & UserMeterEnv + type SaveSecretInput = SecretOwnerContext & { - env: Pick + env: SecretWriteEnv scope: SecretScope name: string value: string description?: string | null sessionExpiresAt?: string | null userEmail?: string | null + waitUntil?: (promise: Promise) => void } type ListSecretsInput = SecretOwnerContext & { @@ -126,6 +130,7 @@ export async function saveSecret( const encryptedValue = await encryptSecretValue(input.env, value) await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -152,6 +157,7 @@ export async function saveSecret( } : null, }), + waitUntil: input.waitUntil, }) const now = new Date().toISOString() await upsertSecretEntry({ @@ -189,13 +195,14 @@ export async function saveSecret( } export async function updateUserSecretForPackage(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null packageId: string name: string value: string description?: string | null + waitUntil?: (promise: Promise) => void }) { const [saved] = await updateUserSecretsForPackageAtomically({ env: input.env, @@ -209,6 +216,7 @@ export async function updateUserSecretForPackage(input: { description: input.description, }, ], + waitUntil: input.waitUntil, }) if (!saved) { throw new Error( @@ -226,7 +234,7 @@ export async function updateUserSecretForPackage(input: { * fail-closes in SQL if any secret lacks the grant. */ export async function setSecretsAtomically(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null secrets: Array<{ @@ -236,6 +244,7 @@ export async function setSecretsAtomically(input: { description?: string | null }> storageContext?: StorageContext | null + waitUntil?: (promise: Promise) => void }): Promise> { if (input.secrets.length === 0) { throw new Error('At least one secret is required.') @@ -279,6 +288,7 @@ export async function setSecretsAtomically(input: { value: secret.value, description: secret.description, })), + waitUntil: input.waitUntil, }) } @@ -293,11 +303,12 @@ export async function setSecretsAtomically(input: { description: secret.description, })), storageContext, + waitUntil: input.waitUntil, }) } export async function updateUserSecretsForPackageAtomically(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null packageId: string @@ -306,6 +317,7 @@ export async function updateUserSecretsForPackageAtomically(input: { value: string description?: string | null }> + waitUntil?: (promise: Promise) => void }): Promise> { const packageId = input.packageId.trim() if (!packageId) throw new Error('Package id is required.') @@ -371,9 +383,11 @@ export async function updateUserSecretsForPackageAtomically(input: { await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: requestedStorageBytes, + waitUntil: input.waitUntil, }) try { @@ -420,7 +434,7 @@ export async function updateUserSecretsForPackageAtomically(input: { } async function saveSecretsAtomically(input: { - env: Pick + env: SecretWriteEnv userId: string userEmail?: string | null scope: SecretScope @@ -430,6 +444,7 @@ async function saveSecretsAtomically(input: { description?: string | null }> storageContext?: StorageContext | null + waitUntil?: (promise: Promise) => void }): Promise> { const bucket = await getOrCreateSecretBucket({ db: input.env.APP_DB, @@ -502,9 +517,11 @@ async function saveSecretsAtomically(input: { } await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: requestedStorageBytes, + waitUntil: input.waitUntil, }) await upsertSecretEntriesAtomically({ diff --git a/packages/worker/src/mcp/values/service.node.test.ts b/packages/worker/src/mcp/values/service.node.test.ts index db0223efa3..fdf7dd1724 100644 --- a/packages/worker/src/mcp/values/service.node.test.ts +++ b/packages/worker/src/mcp/values/service.node.test.ts @@ -6,6 +6,13 @@ import { resolveStorageScopeOrder, } from '#mcp/storage-bindings.ts' import { deleteAllAppScopedValues } from '#worker/package-config-cleanup.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { + createInMemoryUserMeterEnv, + createWaitUntilDrain, + withPatchedDbPrepare, +} from '#worker/test-support/user-meter.ts' import { deleteValue, getValue, listValues, saveValue } from './service.ts' import { type ValueBucketRow, type ValueEntryRow } from './types.ts' @@ -663,3 +670,122 @@ test('saveValue permits underscore-prefixed ordinary value names', async () => { value: '{"provider":"widgets"}', }) }) + +test('saveValue awaits atomic D1 storage reserve; D1→UserMeter shadow is deferred and non-blocking', async () => { + const testDb = createValueTestDb() + const meter = createInMemoryUserMeterEnv() + const drain = createWaitUntilDrain() + const userId = 'a'.repeat(64) + const email = 'value-storage@example.com' + let d1StorageBytes = 100 + let failShadowPointRead = false + await meter.seedStorageBytes({ userId, bytes: 7 }) + + let releaseShadowRead!: () => void + const shadowReadGate = new Promise((resolve) => { + releaseShadowRead = resolve + }) + let shadowPointReadStarted = false + let atomicReserveCompleted = false + using _patch = withPatchedDbPrepare(testDb.db, (originalPrepare) => { + return ((query: string) => { + const normalized = query.replace(/\s+/g, ' ').trim().toLowerCase() + const statement = originalPrepare(query) + const originalBind = statement.bind.bind(statement) + return { + bind(...params: Array) { + const bound = originalBind(...params) + return { + first: async () => { + if ( + normalized.includes('select plan, stripe_plan from users') && + normalized.includes('stable_user_id') + ) { + return { plan: 'pro', stripe_plan: null } as T + } + if ( + normalized.includes('select d1_storage_bytes as bytes') && + normalized.includes('from users') + ) { + if (failShadowPointRead) { + throw new Error('forced D1→UserMeter shadow read failure') + } + shadowPointReadStarted = true + await shadowReadGate + return { bytes: d1StorageBytes } as T + } + return await bound.first() + }, + all: bound.all.bind(bound), + raw: bound.raw?.bind(bound), + run: async () => { + if ( + normalized.includes( + 'set d1_storage_bytes = d1_storage_bytes + ?', + ) + ) { + const requested = Number(params[0]) + const limit = Number(params[4]) + if (d1StorageBytes + requested <= limit) { + d1StorageBytes += requested + atomicReserveCompleted = true + return { meta: { changes: 1 } } + } + return { meta: { changes: 0 } } + } + return await bound.run() + }, + } + }, + } + }) as D1Database['prepare'] + }) + + const env = { APP_DB: testDb.db, ...meter.env } + const saved = await saveValue({ + env, + userId, + userEmail: email, + scope: 'user', + name: 'metered-value', + value: 'hello-storage', + waitUntil: drain.waitUntil, + }) + expect(saved).toMatchObject({ name: 'metered-value' }) + expect(atomicReserveCompleted).toBe(true) + expect(d1StorageBytes).toBeGreaterThan(100) + expect(shadowPointReadStarted).toBe(true) + expect(await userMeterRpc({ env, userId }).readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: 7, + }) + + releaseShadowRead() + await drain.drain() + expect(await userMeterRpc({ env, userId }).readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: d1StorageBytes, + }) + + failShadowPointRead = true + consoleWarn.mockImplementation(() => {}) + const failingDrain = createWaitUntilDrain() + const bytesBeforeFailingSave = d1StorageBytes + await expect( + saveValue({ + env, + userId, + userEmail: email, + scope: 'user', + name: 'metered-value-2', + value: 'still-ok', + waitUntil: failingDrain.waitUntil, + }), + ).resolves.toMatchObject({ name: 'metered-value-2' }) + expect(d1StorageBytes).toBeGreaterThan(bytesBeforeFailingSave) + await failingDrain.drain() + expect(consoleWarn).toHaveBeenCalledWith( + 'entitlement-storage-bytes-shadow-failed', + expect.any(Error), + ) +}) diff --git a/packages/worker/src/mcp/values/service.ts b/packages/worker/src/mcp/values/service.ts index e2becb8bfc..8dc3d81b49 100644 --- a/packages/worker/src/mcp/values/service.ts +++ b/packages/worker/src/mcp/values/service.ts @@ -21,6 +21,7 @@ import { assertWithinStorageBytesEntitlement, estimateEntitlementStorageEntryByteDelta, } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' type ValueOwnerContext = { userId: string @@ -28,13 +29,14 @@ type ValueOwnerContext = { } type SaveValueInput = ValueOwnerContext & { - env: Pick + env: Pick & UserMeterEnv scope: ValueScope name: string value: string description?: string | null sessionExpiresAt?: string | null userEmail?: string | null + waitUntil?: (promise: Promise) => void } type ListValuesInput = ValueOwnerContext & { @@ -87,6 +89,7 @@ export async function saveValue(input: SaveValueInput): Promise { }) await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -104,6 +107,7 @@ export async function saveValue(input: SaveValueInput): Promise { } : null, }), + waitUntil: input.waitUntil, }) const now = new Date().toISOString() await upsertValueEntry({ diff --git a/packages/worker/src/package-registry/service.node.test.ts b/packages/worker/src/package-registry/service.node.test.ts index 14b86f6dc6..83ba2c744e 100644 --- a/packages/worker/src/package-registry/service.node.test.ts +++ b/packages/worker/src/package-registry/service.node.test.ts @@ -1095,6 +1095,9 @@ function createEntitlementsDatabase(input: { const user = users.find((row) => row.email === params[0]) return (user ? { plan: user.plan } : null) as T | null } + if (query.includes('SELECT d1_storage_bytes AS bytes')) { + return null + } if ( query.includes('SELECT COUNT(*) AS count FROM saved_packages') ) { diff --git a/packages/worker/src/package-registry/service.ts b/packages/worker/src/package-registry/service.ts index c0e4790e48..673946208f 100644 --- a/packages/worker/src/package-registry/service.ts +++ b/packages/worker/src/package-registry/service.ts @@ -217,6 +217,7 @@ export async function refreshSavedPackageProjection(input: { packageId: string sourceId: string rebuildArtifacts?: boolean + waitUntil?: (promise: Promise) => void }) { return await withAccountWriteLease({ db: input.env.APP_DB, @@ -248,6 +249,7 @@ export async function refreshSavedPackageProjection(input: { }) await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: input.userEmail, requested: estimateEntitlementStorageEntryByteDelta({ @@ -276,6 +278,7 @@ export async function refreshSavedPackageProjection(input: { } : null, }), + waitUntil: input.waitUntil, }) if (existing) { await updateSavedPackage(input.env.APP_DB, { diff --git a/packages/worker/src/storage-runner.entitlement.node.test.ts b/packages/worker/src/storage-runner.entitlement.node.test.ts index 5b591fea55..d7c51bb55a 100644 --- a/packages/worker/src/storage-runner.entitlement.node.test.ts +++ b/packages/worker/src/storage-runner.entitlement.node.test.ts @@ -229,3 +229,52 @@ test('an unreachable peer bucket with a stored estimate never blocks a write', a // Only the write target was measured live; no peer fan-out happened. expect(probedStorageIds).toEqual(['package:target']) }) + +test('StorageRunner baseline composes D1 payload with bucket estimates via check-only getCurrent', async () => { + const { planLimits } = await import('#worker/entitlements/plans.ts') + const { isEntitlementLimitError } = + await import('#worker/entitlements/errors.ts') + const userId = 'user-1' + const limit = planLimits.free.maxStorageBytes + mockModule.readUserD1StorageBytes.mockImplementation( + async (input: { userId: string }) => { + expect(input.userId).toBe(userId) + return limit - 100 + }, + ) + mockModule.listUserStorageBucketEstimates.mockResolvedValue([ + { storageId: 'package:peer', estimatedBytes: 50 }, + { storageId: 'package:target', estimatedBytes: 200 }, + ]) + const probedStorageIds: Array = [] + const getEstimatedBytes = async (storageId: string) => { + probedStorageIds.push(storageId) + return { estimatedBytes: storageId === 'package:target' ? 60 : 50 } + } + + const denied = await assertStorageRunnerWriteWithinEntitlement({ + env: createEstimateEnv(getEstimatedBytes), + userId, + email: null, + storageId: 'package:target', + requested: 1, + }).then( + () => null, + (thrown: unknown) => thrown, + ) + expect(isEntitlementLimitError(denied)).toBe(true) + expect(denied).toMatchObject({ + details: { + resource: 'storage_bytes', + current: limit - 100 + 50 + 60, + limit, + }, + }) + expect(probedStorageIds).toEqual(['package:target']) + expect(mockModule.readUserD1StorageBytes).toHaveBeenCalledWith( + expect.objectContaining({ + userId, + db: expect.anything(), + }), + ) +}) diff --git a/packages/worker/src/storage-runner.workers.test.ts b/packages/worker/src/storage-runner.workers.test.ts index 5f9859b213..c606da2f18 100644 --- a/packages/worker/src/storage-runner.workers.test.ts +++ b/packages/worker/src/storage-runner.workers.test.ts @@ -285,6 +285,7 @@ test('storage runner storage byte entitlement aggregates only inventoried user b .bind(targetD1Bytes, new Date().toISOString(), userId) .run() expect(initialD1Bytes).toBe(0) + // D1 remains the sole payload authority for composed getCurrent. await expect( readUserD1StorageBytes({ db: env.APP_DB, userId }), ).resolves.toBe(targetD1Bytes) diff --git a/packages/worker/src/test-support/user-meter.ts b/packages/worker/src/test-support/user-meter.ts index bf3a390f23..c02fe7f3b5 100644 --- a/packages/worker/src/test-support/user-meter.ts +++ b/packages/worker/src/test-support/user-meter.ts @@ -2,10 +2,12 @@ import { isDailyEntitlementResource, userMeterMirrorUpdatedAtToken, type DailyEntitlementResource, + type UserMeterStorageBytesState, } from '#worker/entitlements/user-meter-do.ts' import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' type MeterRow = { count: number; revision: number } +type StorageRow = { bytes: number; revision: number; updatedAt: string } /** * In-memory UserMeter stub keyed by `idFromName` (stable userId) for node-unit @@ -14,6 +16,7 @@ type MeterRow = { count: number; revision: number } */ export function createInMemoryUserMeterEnv() { const metersByUser = new Map>() + const storageByUser = new Map() function counterKey(resource: string, day: string) { return `${resource}\0${day}` @@ -37,6 +40,15 @@ export function createInMemoryUserMeterEnv() { } } + function storageReady(row: StorageRow) { + return { + outcome: 'ready' as const, + bytes: row.bytes, + revision: row.revision, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(row.revision), + } + } + return { async initialize(input: { resource: string @@ -105,8 +117,69 @@ export function createInMemoryUserMeterEnv() { rows.set(counterKey(input.resource, input.day), next) return ready(next) }, + async initializeStorageBytes(input: { + bytes: number + updatedAt: string + }) { + const existing = storageByUser.get(userId) + if (existing) return { ...storageReady(existing), created: false } + const row = { + bytes: Math.max(0, Math.trunc(input.bytes) || 0), + revision: 1, + updatedAt: input.updatedAt, + } + storageByUser.set(userId, row) + return { ...storageReady(row), created: true } + }, + async readStorageBytes() { + const existing = storageByUser.get(userId) + if (!existing) return { outcome: 'needs_bootstrap' as const } + return storageReady(existing) + }, + async reserveStorageBytes(input: { + requested: number + limit: number + updatedAt: string + }) { + const requested = Math.max(0, Math.trunc(input.requested) || 0) + const existing = storageByUser.get(userId) + if (!existing) return { outcome: 'needs_bootstrap' as const } + if ( + requested > 0 && + (input.limit < 1 || existing.bytes + requested > input.limit) + ) { + return { ...storageReady(existing), reserved: false } + } + if (requested === 0) { + return { ...storageReady(existing), reserved: true } + } + const next = { + bytes: existing.bytes + requested, + revision: existing.revision + 1, + updatedAt: input.updatedAt, + } + storageByUser.set(userId, next) + return { ...storageReady(next), reserved: true } + }, + async setStorageBytes(input: { bytes: number; updatedAt: string }) { + const bytes = Math.max(0, Math.trunc(input.bytes) || 0) + const existing = storageByUser.get(userId) + if (!existing) { + const row = { bytes, revision: 1, updatedAt: input.updatedAt } + storageByUser.set(userId, row) + return { ...storageReady(row), created: true } + } + const next = { + bytes, + revision: existing.revision + 1, + updatedAt: input.updatedAt, + } + storageByUser.set(userId, next) + return { ...storageReady(next), created: false } + }, async purge() { rows.clear() + storageByUser.delete(userId) return { ok: true as const } }, async exportCounters() { @@ -121,7 +194,21 @@ export function createInMemoryUserMeterEnv() { mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(row.revision), } }) - return { counters, nextStartAfter: null, truncated: false } + const storage = storageByUser.get(userId) + const storageBytesShadow: UserMeterStorageBytesState | null = storage + ? { + bytes: storage.bytes, + revision: storage.revision, + updatedAt: storage.updatedAt, + mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(storage.revision), + } + : null + return { + counters, + storageBytesShadow, + nextStartAfter: null, + truncated: false, + } }, } } @@ -136,6 +223,7 @@ export function createInMemoryUserMeterEnv() { return { env, metersByUser, + storageByUser, async seed(input: { userId: string resource: DailyEntitlementResource @@ -149,6 +237,16 @@ export function createInMemoryUserMeterEnv() { updatedAt: new Date().toISOString(), }) }, + async seedStorageBytes(input: { + userId: string + bytes: number + updatedAt?: string + }) { + await meterFor(input.userId).initializeStorageBytes({ + bytes: input.bytes, + updatedAt: input.updatedAt ?? new Date().toISOString(), + }) + }, } } From eaaa50906609ef9abf196ff61a49603d97ac5140 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 06:56:37 +0000 Subject: [PATCH 14/15] docs(entitlements): inventory storage shadow Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 104 ++++---- .../contributing/architecture/entitlements.md | 222 ++++++++++++------ .../contributing/architecture/primitives.yaml | 4 +- .../worker/src/account/export.node.test.ts | 14 +- packages/worker/src/account/export.ts | 11 +- .../worker/src/account/user-owned-surfaces.ts | 2 +- .../src/app/account-usage-data.node.test.ts | 29 ++- 7 files changed, 263 insertions(+), 123 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 504de9bbdc..8e74c6138e 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -166,13 +166,14 @@ migration-safe chunked interface: with `section: "storage_runner"` and a `storage_id`, using the same StorageRunner `exportStorage({ pageSize, startAfter })` RPC as the dedicated storage export capability. User meter counters use `section: "user_meter"` and - the `UserMeter.exportCounters` RPC. R2 raw MIME, attachment, avatar, and icon - objects use `section: "r2_object"`; each response contains at most one 256 KiB - base64 chunk and an opaque cursor. Each request uses bounded `LIMIT 1` - ownership queries rather than reconstructing inventory. Continuation cursors - bind the source row, object key, size, and ETag; ownership/key mutations and - object overwrites are reported instead of mixing generations. Missing objects - are represented explicitly. + the `UserMeter.exportCounters` RPC (daily counters plus additive + `storageBytesShadow` when present; explicitly non-authoritative). R2 raw MIME, + attachment, avatar, and icon objects use `section: "r2_object"`; each response + contains at most one 256 KiB base64 chunk and an opaque cursor. Each request + uses bounded `LIMIT 1` ownership queries rather than reconstructing inventory. + Continuation cursors bind the source row, object key, size, and ETag; + ownership/key mutations and object overwrites are reported instead of mixing + generations. Missing objects are represented explicitly. D1 manifest counts use bounded SQL `COUNT(*)` queries. D1 section rows are read with SQL-level keyset pagination: every query orders by the table's `rowid`, @@ -196,8 +197,12 @@ Durable Object export behavior: never pruned by retention. See [Run records](./run-records.md). - `UserMeter` exports daily entitlement counter rows through the `user_meter` section (`exportCounters` RPC; keyset pagination by UTC `day` and `resource`). - Retention is self-enforced inside the DO (seven UTC days of counter and - inbound-delivery-claim rows); see + The same RPC may return additive `storageBytesShadow` when the schema-v4 + shadow row exists (`null` when never shadowed); section totals count it as one + row when present, but it is explicitly **non-authoritative** — usage and + enforcement read D1 `users.d1_storage_bytes`. Retention is self-enforced + inside the DO (seven UTC days of counter and inbound-delivery-claim rows); + shadow storage-byte state is not time-pruned. See [Entitlements](./entitlements.md#usermeter-expand-phase). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. @@ -238,11 +243,15 @@ The schema is defined by migrations in `packages/worker/migrations/`: default `public`) come from migration 0068. `account_type` (`'person'` default or `'platform'`, migration 0072) distinguishes normal signups from operator-provisioned platform accounts that own official package scopes (see - [Platform accounts](./platform-accounts.md)). Inbound email routing does not - reverse-resolve stable ids at all — it uses the indexed username lookup - (`findPublicUserIdentityByUsername`). Contextless paths resolve stable ids - with one indexed point read on `users.stable_user_id` (for example - `findUserAccountByStableUserId`). + [Platform accounts](./platform-accounts.md)). `d1_storage_bytes` and + `d1_storage_bytes_updated_at` (migration 0122) are the **sole authority** for + D1 payload storage-byte read, enforcement, and reconciliation. UserMeter + `storage_bytes_state` (schema v4) is an optional expand-phase shadow only — + see [Entitlements](./entitlements.md#usermeter-expand-phase). Inbound email + routing does not reverse-resolve stable ids at all — it uses the indexed + username lookup (`findPublicUserIdentityByUsername`). Contextless paths + resolve stable ids with one indexed point read on `users.stable_user_id` (for + example `findUserAccountByStableUserId`). - `platform_feedback`: attributed, user-approved Kody feedback and admin triage state. Submitter identity remains on the row; optional reviewer attribution is cleared if that admin account is deleted. Open and triaged rows remain until @@ -489,16 +498,20 @@ Storage split: Daily rate-style entitlement counters and inbound email delivery-id idempotency live in a per-user `UserMeter` Durable Object with SQLite -(`packages/worker/src/entitlements/user-meter-do.ts`). The Worker binding is -`USER_METER` (class `UserMeter`; Wrangler SQLite migration tag `v21` via -`new_sqlite_classes` in `packages/worker/wrangler.jsonc`). +(`packages/worker/src/entitlements/user-meter-do.ts`). Schema v4 adds an +optional `storage_bytes_state` singleton as a **best-effort shadow** of D1 +`users.d1_storage_bytes` for future cutover — D1 remains sole authority for +reads, reserves, and reconciliation in the current additive slice. The Worker +binding is `USER_METER` (class `UserMeter`; Wrangler SQLite migration tag `v21` +via `new_sqlite_classes` in `packages/worker/wrangler.jsonc`). Naming matches `RunLog` and `JobManager`: one object per untrimmed stable MCP `userId` via `userMeterDurableObjectName(userId)` → `idFromName(userId)` in `packages/worker/src/user-scoped-durable-object-name.ts`. There is no `user_id` column inside the DO because the object identity is the user. -SQLite ownership (schema version tracked in `user_meter_meta`): +SQLite ownership (schema version tracked in `user_meter_meta`; current version +**4**): - `daily_counters` — authoritative UTC-day counters for `email_sends_per_day`, `email_receives_per_day`, `execute_calls_per_day`, and @@ -509,30 +522,40 @@ SQLite ownership (schema version tracked in `user_meter_meta`): claim's resource/day, post-charge counter, revision, and `claimed_at` so Cloudflare Email Routing retries inside the 48-hour inbound dedupe window cannot double-charge `email_receives_per_day`. +- `storage_bytes_state` — singleton **shadow** of D1 payload bytes (`id = 1` + CHECK constraint, `bytes`, monotonic `revision`, `updated_at`). Added in + schema v4. Populated by optional non-awaited shadow writes after D1 reserves + and optional reconcile shadows; never read for enforcement or usage display. + StorageRunner bucket estimates stay outside this row (see + [Entitlements](./entitlements.md#usermeter-expand-phase)). Retention is self-enforced inside the DO: every read/write path opportunistically deletes counter and claim rows older than seven UTC days (`userMeterDailyCounterRetentionDays`). Enforcement only needs the current day; the window covers timezone edge cases, recent account exports, and inbound -retries. - -**Expand-phase D1 mirror:** enforcement and point reads are authoritative in -UserMeter. D1 `entitlement_daily_counters` is **not** dropped — it remains a -best-effort mirror for existing readers and reporting. After each DO -consume/refund/inbound claim, the entitlements service schedules a non-awaited -absolute mirror write keyed by `(user_id, resource, day)` with a -revision-ordered `updated_at` token (`r/` + zero-padded revision from -`userMeterMirrorUpdatedAtToken`) so late writes cannot overwrite newer state. -See [Entitlements](./entitlements.md#usermeter-expand-phase). - -**Cold bootstrap:** a missing `(resource, day)` row returns `needs_bootstrap`. -The service performs one legacy D1 point read on `entitlement_daily_counters`, -then `initialize()` seeds the DO row with `INSERT OR IGNORE` (concurrent callers -cannot double-apply the baseline). Warm paths never read D1 for enforcement. - -Account deletion calls `UserMeter.purge()` (one RPC per user, no D1 id scan). -Account export pages `UserMeter.exportCounters` through the `user_meter` -manifest section / `account_export_section`. +retries. Shadow storage-byte state is not time-pruned. + +**Expand-phase D1 mirrors (daily counters only):** enforcement and point reads +are authoritative in UserMeter for daily counters. D1 +`entitlement_daily_counters` is **not** dropped — it remains a best-effort +mirror for existing readers and reporting. After each DO consume/refund/inbound +claim, the entitlements service schedules a non-awaited absolute mirror write +keyed by `(user_id, resource, day)` with a revision-ordered `updated_at` token +(`r/` + zero-padded revision from `userMeterMirrorUpdatedAtToken`) so late +writes cannot overwrite newer state. See +[Entitlements](./entitlements.md#usermeter-expand-phase). + +**Daily cold bootstrap:** a missing `(resource, day)` row returns +`needs_bootstrap`. The service performs one legacy D1 point read on +`entitlement_daily_counters`, then `initialize()` seeds the DO row with +`INSERT OR IGNORE` (concurrent callers cannot double-apply the baseline). Warm +daily paths never read D1 for enforcement. + +Account deletion calls `UserMeter.purge()` (one RPC per user, no D1 id scan; +`deleteAll` clears counters, claims, and any shadow storage state). Account +export pages `UserMeter.exportCounters` through the `user_meter` manifest +section / `account_export_section` (daily counters plus additive +`storageBytesShadow` when present; shadow field is non-authoritative). ### Package state model @@ -570,8 +593,9 @@ via `durableObjectNameFromParts`); domain helpers such as and dedicated unpruned state (workflow projections, job-run observability, package activation counters/milestones). See [Run records](./run-records.md). - `UserMeter` — `userMeterDurableObjectName(userId)` → `idFromName(userId)`. One - daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`). - See [Entitlements](./entitlements.md#usermeter-expand-phase). + daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`), + plus optional schema-v4 D1 storage-byte shadow. See + [Entitlements](./entitlements.md#usermeter-expand-phase). - `McpClientHub` — `mcpClientHubDurableObjectName(userId)` → `idFromName(userId.trim())`. - `StorageRunner` — `storageRunnerDurableObjectName(userId, storageId)` → @@ -886,7 +910,7 @@ to `durableObjectNameFromParts`). - `JobManager`: `idFromName(userId)` (no trim). - `RunLog`: `idFromName(userId)` (no trim); one execution-history DO per user. - `UserMeter`: `idFromName(userId)` (no trim); one daily-entitlement meter DO - per user. + per user, plus optional schema-v4 D1 storage-byte shadow. - `McpClientHub`: `idFromName(userId.trim())`. - `StorageRunner`: `idFromName(JSON.stringify([userId, storageId]))`. - `RepoSession`: `idFromName(repo_sessions.id)`; the key is not user-prefixed, diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index c4c983ba17..2879002bd5 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -119,6 +119,17 @@ per-user `UserMeter` Durable Object** (`USER_METER` binding). Code lives in `packages/worker/src/entitlements/user-meter-do.ts` and `user-meter-client.ts`; storage layout and naming are documented in [Data storage](./data-storage.md). +**D1 payload storage bytes** (`storage_bytes`) stay authoritative on D1 +`users.d1_storage_bytes` (migration 0122). UserMeter schema v4 adds an optional +`storage_bytes_state` singleton as a **best-effort shadow** for future cutover — +it does not drive reads, reserves, or reconciliation in this additive slice. + +StorageRunner bucket `estimatedBytes` and the per-bucket inventory in +`user_storage_buckets` stay a **separate** quota component. StorageRunner write +chokepoints pass `getCurrent` as a check-only composed total (D1 payload bytes +from `readUserD1StorageBytes` plus bucket estimates); that path does **not** +reserve bytes in D1 or UserMeter. + **Strong enforcement:** `consumeDailyEntitlement` and inbound `consumeInboundDelivery` RPCs check the plan limit and increment inside the DO. The Durable Object request model serializes mutations per user; counter updates @@ -167,15 +178,71 @@ mirror is scheduled from the email path via `scheduleAbsoluteDailyEntitlementMirror` so the email subsystem does not duplicate mirror SQL. -**Remaining migration dependency:** retiring the D1 mirror and -`entitlement_daily_counters` table waits until reporting-off-D1 work merges and -mirror parity is verified in production. This expand phase intentionally does -**not** move `storage_bytes` enforcement, `package_service_states` running -counts, or account-deletion write fencing to UserMeter. +### D1 payload storage bytes — UserMeter shadow (expand phase slice 3) + +D1 `users.d1_storage_bytes` and `users.d1_storage_bytes_updated_at` remain the +**sole read, enforcement, and reconciliation authority** for the D1 payload +component of `storage_bytes`. Every caller — including email — reserves through +the same atomic D1 `UPDATE` in `assertWithinStorageBytesEntitlement`; denial +semantics and `EntitlementLimitError` always come from D1. + +UserMeter schema **v4** adds a singleton `storage_bytes_state` row (`id = 1`) as +**best-effort shadow / future-cutover support only**. The DO row is never read +for usage display or enforcement in this slice. + +**Optional non-email shadow writes:** DO-backed write chokepoints (values, +secrets, memories, and saved-package projections) may pass optional `env` into +`assertWithinStorageBytesEntitlement`. After a **successful D1 reserve**, the +service schedules a non-awaited best-effort shadow via `waitUntil` when +available: re-read the latest D1 value, then absolute-set UserMeter through +`setStorageBytes`. Shadow tasks re-read D1 at execution time so a delayed shadow +cannot apply a stale absolute and leave the DO behind D1. Missing `USER_METER` +or shadow failures log `entitlement-storage-bytes-shadow-failed` and never +affect D1 enforcement. + +**Optional reconcile shadow:** `reconcileUserD1StorageBytes` absolute-sets D1 +from `calculateUserD1StorageBytes` and awaits that write. When optional `env` is +present, it best-effort shadows the post-update D1 value into UserMeter; shadow +failures are logged and never fail or poison the lane. Current +`d1_storage_reconciliation` wiring passes `{ db }` only. + +**Usage reads stay on D1:** account usage, admin drill-down, and +`readCurrentEntitlementResourceUsage` for `storage_bytes` call +`readUserD1StorageBytes` only. + +**Account export and purge:** `UserMeter.exportCounters` may return additive +non-authoritative `storageBytesShadow` when the shadow row exists (counted in +the `user_meter` section total). `UserMeter.purge()` clears counters, inbound +delivery claims, and any shadow storage state via `deleteAll`. + +### Future storage authority flip (contract follow-up) + +A separate contract PR — not a merge blocker for this additive slice — will flip +D1 payload storage-byte authority into UserMeter once: + +1. **mailbox-do** passes `env` on email inbound/outbound storage reservations, + and +2. **cron-restructure** passes `env` into `d1_storage_reconciliation` lane + wiring in `index.ts`. + +Only then do reads, reserves, and reconciliation switch to UserMeter-first with +D1 as mirror/cursor. Until that flip, shadow divergence is acceptable; D1 +remains the contract. + +**Remaining UserMeter expand roadmap:** slice 4 — `package_service_states` +running counts (service liveness); slice 5 — account-deletion write fencing. +Storage authority flip is tracked separately as the contract follow-up above. + +**Daily-counter mirror retirement:** dropping D1 `entitlement_daily_counters` +waits until reporting-off-D1 work merges and mirror parity is verified in +production. Module wiring: `consumeDailyEntitlement`, `refundDailyEntitlement`, and -`readDailyEntitlementResourceUsage` in `service.ts` require `env.USER_METER` and -fail closed when the binding is missing. +`readDailyEntitlementResourceUsage` require `env.USER_METER` and fail closed +when the binding is missing. Storage-byte helpers (`readUserD1StorageBytes`, +`assertWithinStorageBytesEntitlement`, `reconcileUserD1StorageBytes`) treat +`USER_METER` as optional shadow-only; missing binding is a no-op for shadow +scheduling. ## Schema history @@ -351,60 +418,65 @@ Rules: against the limit instead of an accumulating count: the enforcement point passes the candidate size via `getCurrent` with `requested: 0`. There is no built-in counter for these. -- **Storage-byte limits** (`storage_bytes`) store the D1 payload estimate on - `users.d1_storage_bytes` (migration 0122). Entitlement checks read that - indexed user row and D1 write chokepoints atomically reserve their existing - positive byte-delta estimate before the write, so mailbox growth and other hot - writes never rescan the user's accumulated rows. The estimate covers - user-owned rows with durable payloads (`email_messages.raw_size` plus - extracted message bodies/metadata, externally stored attachments, values, - encrypted secrets, memories, saved-package projections, jobs, repo/session - metadata, package invocation results, and published artifact metadata). Run - records in the per-user `RunLog` Durable Object are intentionally **excluded** - from the quota — they are observability history, not user content. A fixed - batch of eight oldest users is reconciled every five-minute cron tick by - `d1_storage_reconciliation`; each selected counter is replaced with the - authoritative cross-surface sum. This bounded lane corrects over-reservation - after failed writes, shrinkage from deletes, and writes on surfaces without a - storage-byte chokepoint. Failed rows retain their conservative value and - rotate to the back of the queue for a later retry. StorageRunner Durable - Object buckets expose their own `estimatedBytes`, and each bucket's latest - measurement is persisted on its `user_storage_buckets.estimated_bytes` - inventory row (migration 0118). Write chokepoints pass `getCurrent` as - `D1 counter + sum of per-bucket estimates`, where only the bucket that - triggers the baseline read (plus any inventoried bucket with no stored - estimate yet) is probed live; every other bucket contributes its stored D1 - estimate, so mutating writes do not fan `getEstimatedBytes` RPCs across the - user's whole bucket inventory. Live probe results are persisted - fire-and-forget with **UPDATE-only** statements (they can never recreate an - inventory row removed by account, package, or job deletion), and mutating - StorageRunner RPCs opportunistically refresh their own bucket's stored - estimate after the write, throttled per bucket per isolate - (`storageBucketEstimateRefreshMinIntervalMs`). Stored estimates are therefore - freshness hints with bounded lag — acceptable for an order-of-magnitude cap - because the bucket paying the baseline read is measured live and the run cache - accounts for the run's own accepted writes. `requested` is the candidate - payload size when known. Pure read-only `storage.sql` / `packageStorage().sql` - statements (`SELECT` / `EXPLAIN` / schema `PRAGMA`) skip the baseline read - entirely even when the helper marks the call writable. Mutating SQL and - `storage.set` in one sandbox share a per-run baseline cache so repeated writes - do not re-read the baseline; a later write in the same run that targets a - **different** already-inventoried bucket reuses that bucket's stored estimate - rather than probing it live (bounded staleness, same trade-off as peers). Each - live estimate read waits at most ~2s via `Promise.race` and is retried with - backoff (`storageEstimateReadRetryDelaysMs`; a single 150ms retry lost to - transient per-bucket DO read failures in production) before failing closed for - the caller; the underlying DO RPC is not cancelled if the runtime keeps it - running. A cron lane (`storage_bucket_estimate_backfill`, - `packages/worker/src/storage-buckets/estimate-backfill.ts`) sweeps inventory - rows without a stored estimate in bounded batches (failed probes stay - unmeasured and are retried on later sweeps) so freshly migrated inventories - converge to stored estimates within a few ticks instead of making each user's - first mutating write pay (and possibly fail on) the whole-inventory probe. The - counter intentionally does **not** attempt to scan Cloudflare Artifacts - repository contents, KV snapshot/bundle bodies, R2 object listings beyond - `email_messages.raw_size`, or Vectorize: those stores either lack reliable - byte metadata or are derived from D1 and are documented in `data-storage.md`. +- **Storage-byte limits** (`storage_bytes`) split into two quota components: + 1. **D1 payload bytes (authoritative on D1):** user-owned D1 rows with durable + payloads (`email_messages.raw_size` plus extracted message bodies/metadata, + externally stored attachments, values, encrypted secrets, memories, + saved-package projections, jobs, repo/session metadata, package invocation + results, and published artifact metadata). Run records in the per-user + `RunLog` Durable Object are intentionally **excluded** — they are + observability history, not user content. Write chokepoints atomically + reserve positive byte deltas via `assertWithinStorageBytesEntitlement` + against `users.d1_storage_bytes` (every caller, including email). Optional + `env` on non-email paths schedules a best-effort UserMeter shadow after a + successful D1 reserve; shadow state is not read for enforcement or usage. + The bounded `d1_storage_reconciliation` lane recomputes the authoritative + cross-surface sum via `calculateUserD1StorageBytes` and absolute-sets D1; + optional `env` shadows into UserMeter after the D1 write — see + [UserMeter (expand phase)](#usermeter-expand-phase). + + 2. **StorageRunner bucket estimates (separate):** each bucket exposes + `estimatedBytes`; inventory rows persist the latest measurement on + `user_storage_buckets.estimated_bytes` (migration 0118). Write chokepoints + that pass `getCurrent` compose + `D1 payload (readUserD1StorageBytes) + sum of per-bucket estimates` for a + **check-only** entitlement comparison — `getCurrent` never reserves in D1 + or UserMeter. Only the bucket that triggers the baseline read (plus any + inventoried bucket with no stored estimate yet) is probed live; every other + bucket contributes its stored D1 estimate, so mutating writes do not fan + `getEstimatedBytes` RPCs across the whole inventory. Live probe results are + persisted fire-and-forget with **UPDATE-only** statements (they can never + recreate an inventory row removed by account, package, or job deletion), + and mutating StorageRunner RPCs opportunistically refresh their own + bucket's stored estimate after the write, throttled per bucket per isolate + (`storageBucketEstimateRefreshMinIntervalMs`). Stored estimates are + freshness hints with bounded lag — acceptable for an order-of-magnitude cap + because the bucket paying the baseline read is measured live and the run + cache accounts for the run's own accepted writes. `requested` is the + candidate payload size when known. Pure read-only `storage.sql` / + `packageStorage().sql` statements (`SELECT` / `EXPLAIN` / schema `PRAGMA`) + skip the baseline read entirely even when the helper marks the call + writable. Mutating SQL and `storage.set` in one sandbox share a per-run + baseline cache so repeated writes do not re-read the baseline; a later + write in the same run that targets a **different** already-inventoried + bucket reuses that bucket's stored estimate rather than probing it live + (bounded staleness, same trade-off as peers). Each live estimate read waits + at most ~2s via `Promise.race` and is retried with backoff + (`storageEstimateReadRetryDelaysMs`; a single 150ms retry lost to transient + per-bucket DO read failures in production) before failing closed for the + caller; the underlying DO RPC is not cancelled if the runtime keeps it + running. A cron lane (`storage_bucket_estimate_backfill`, + `packages/worker/src/storage-buckets/estimate-backfill.ts`) sweeps + inventory rows without a stored estimate in bounded batches (failed probes + stay unmeasured and are retried on later sweeps) so freshly migrated + inventories converge to stored estimates within a few ticks instead of + making each user's first mutating write pay (and possibly fail on) the + whole-inventory probe. The D1 payload counter intentionally does **not** + attempt to scan Cloudflare Artifacts repository contents, KV + snapshot/bundle bodies, R2 object listings beyond + `email_messages.raw_size`, or Vectorize: those stores either lack reliable + byte metadata or are derived from D1 and are documented in + `data-storage.md`. ### Concurrency @@ -458,20 +530,20 @@ Use `getCurrent` only when the built-in D1 counter cannot express the resource. ## Enforcement points -| Resource | Enforcement point | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `scheduled_jobs` | `createJob` in `packages/worker/src/jobs/service.ts` (exemplar) | -| `saved_packages` | new-package branch of `package_save` and projection insert | -| `package_services` | `service_start` capability path (count from `package_service_states`) | -| `persistent_package_services` | `service_start` for services declared `mode: 'persistent'` | -| `repo_sessions` | `repo_open_session` before creating a new session | -| `email_sends_per_day` | `sendOutboundEmail` (`consumeDailyEntitlement`; plan limit from `resolvePlanLimit`) | -| `email_receives_per_day` | `handleInboundEmail` (`consumeDailyEntitlement`; same plan limits; refund only on `RetryableInboundStorageError`) | -| `stored_email_messages` | `handleInboundEmail` before storage (`assertWithinEntitlement`; `max` caps from `planLimits.max`) | -| `email_message_bytes` | `handleInboundEmail` before quota/parse (per-message raw size via `resolveEmailResourceLimit`) | -| `secrets` | new-entry branch of `saveSecret` in `packages/worker/src/mcp/secrets/service.ts` | -| `concurrent_workflows` | `createDynamicCallableWorkflow` (`reserveWorkflowProjectionSlot` + `assertWithinEntitlement` getCurrent; `max` = 5,000) | -| `storage_bytes` | D1 payload writes (values, secrets, memories, saved-package projections, email storage) and StorageRunner write tools/app RPCs | +| Resource | Enforcement point | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `scheduled_jobs` | `createJob` in `packages/worker/src/jobs/service.ts` (exemplar) | +| `saved_packages` | new-package branch of `package_save` and projection insert | +| `package_services` | `service_start` capability path (count from `package_service_states`) | +| `persistent_package_services` | `service_start` for services declared `mode: 'persistent'` | +| `repo_sessions` | `repo_open_session` before creating a new session | +| `email_sends_per_day` | `sendOutboundEmail` (`consumeDailyEntitlement`; plan limit from `resolvePlanLimit`) | +| `email_receives_per_day` | `handleInboundEmail` (`consumeDailyEntitlement`; same plan limits; refund only on `RetryableInboundStorageError`) | +| `stored_email_messages` | `handleInboundEmail` before storage (`assertWithinEntitlement`; `max` caps from `planLimits.max`) | +| `email_message_bytes` | `handleInboundEmail` before quota/parse (per-message raw size via `resolveEmailResourceLimit`) | +| `secrets` | new-entry branch of `saveSecret` in `packages/worker/src/mcp/secrets/service.ts` | +| `concurrent_workflows` | `createDynamicCallableWorkflow` (`reserveWorkflowProjectionSlot` + `assertWithinEntitlement` getCurrent; `max` = 5,000) | +| `storage_bytes` | D1 payload writes via `assertWithinStorageBytesEntitlement` (D1 reserve for all callers; optional `env` shadow on non-email paths) and StorageRunner write tools/app RPCs (`getCurrent` check-only for bucket component) | ## Billing diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index a86960099b..5302603cd6 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -513,8 +513,8 @@ primitives: group: storage name: User meter summary: - Per-user UserMeter DO SQLite for daily entitlement counters and inbound - delivery idempotency. + Per-user UserMeter DO SQLite for daily entitlement counters, inbound + delivery idempotency, and expand-phase D1 storage-byte shadow. code: - packages/worker/src/entitlements/user-meter-do.ts - packages/worker/src/entitlements/user-meter-client.ts diff --git a/packages/worker/src/account/export.node.test.ts b/packages/worker/src/account/export.node.test.ts index 08e9964d21..97a962c81c 100644 --- a/packages/worker/src/account/export.node.test.ts +++ b/packages/worker/src/account/export.node.test.ts @@ -1572,6 +1572,12 @@ test('account export includes user_meter counters, pages them, and warns on trun mirrorUpdatedAt: 'r/00000000000000000001', }, ] + const storageBytesShadow = { + bytes: 4_096, + revision: 3, + updatedAt: '2026-07-31T03:00:00.000Z', + mirrorUpdatedAt: 'r/00000000000000000003', + } const exportCounters = vi.fn( async (input: { pageSize?: number; startAfter?: string | null }) => { const pageSize = input.pageSize ?? 100 @@ -1584,6 +1590,7 @@ test('account export includes user_meter counters, pages them, and warns on trun const truncated = startIndex + pageSize < counters.length return { counters: page, + storageBytesShadow, nextStartAfter: truncated ? `${page.at(-1)!.day}:${page.at(-1)!.resource}` : null, @@ -1606,9 +1613,10 @@ test('account export includes user_meter counters, pages them, and warns on trun mcpUserId: 'user-aaa', }) expect(idFromName).toHaveBeenCalledWith('user-aaa') - expect(accountExport.manifest.sections.user_meter?.count).toBe(3) + expect(accountExport.manifest.sections.user_meter?.count).toBe(4) expect(accountExport.durableObjects.userMeter).toEqual({ counters, + storageBytesShadow, nextStartAfter: null, truncated: false, }) @@ -1626,6 +1634,7 @@ test('account export includes user_meter counters, pages them, and warns on trun pageSize: 2, }) expect(first.items).toEqual(counters.slice(0, 2)) + expect(first.storageBytesShadow).toEqual(storageBytesShadow) expect(first.truncated).toBe(true) expect(first.nextStartAfter).toBe('2026-07-30:execute_calls_per_day') @@ -1638,12 +1647,14 @@ test('account export includes user_meter counters, pages them, and warns on trun startAfter: first.nextStartAfter ?? undefined, }) expect(second.items).toEqual(counters.slice(2)) + expect(second.storageBytesShadow).toEqual(storageBytesShadow) expect(second.truncated).toBe(false) expect(second.nextStartAfter).toBeNull() expect(exportCounters).toHaveBeenCalled() exportCounters.mockImplementation(async () => ({ counters: [counters[0]!], + storageBytesShadow: null, nextStartAfter: 'cursor-more', truncated: true, })) @@ -1653,6 +1664,7 @@ test('account export includes user_meter counters, pages them, and warns on trun mcpUserId: 'user-aaa', }) expect(truncatedExport.durableObjects.userMeter?.truncated).toBe(true) + expect(truncatedExport.manifest.sections.user_meter?.count).toBe(1) expect(truncatedExport.manifest.warnings).toContain( 'User meter counters were truncated in the full export; use account_export_section with section "user_meter" to retrieve additional pages.', ) diff --git a/packages/worker/src/account/export.ts b/packages/worker/src/account/export.ts index c3bf88a67e..297abc2c38 100644 --- a/packages/worker/src/account/export.ts +++ b/packages/worker/src/account/export.ts @@ -204,6 +204,10 @@ export type AccountExportArtifactRepo = { type RunRecordsExportPayload = Awaited> +function countUserMeterExportEntries(result: UserMeterExportResult): number { + return result.counters.length + (result.storageBytesShadow == null ? 0 : 1) +} + function countRunRecordsExportEntries( runRecords: RunRecordsExportPayload | null | undefined, ) { @@ -279,6 +283,8 @@ export type AccountExportSectionResult = { nextStartAfter: string | null pageSize: number warnings: Array + /** Non-authoritative UserMeter storage-byte shadow; not part of paging. */ + storageBytesShadow?: UserMeterExportResult['storageBytesShadow'] } function normalizePageSize(pageSize: number | undefined) { @@ -1050,7 +1056,7 @@ async function collectManifestInventoryCounts(input: { }).exportCounters({ pageSize: maxExportPageSize, }) - return page.counters.length + return countUserMeterExportEntries(page) }), safeCount('remote connectors', async () => countScalar( @@ -1571,7 +1577,7 @@ function buildManifest(input: { count: input.durableObjects?.userMeter == null ? (input.inventoryCounts?.userMeterCounters ?? 0) - : input.durableObjects.userMeter.counters.length, + : countUserMeterExportEntries(input.durableObjects.userMeter), warnings: input.warnings.filter( (warning) => warning.startsWith('User meter ') || warning.startsWith('USER_METER '), @@ -1911,6 +1917,7 @@ export async function readAccountExportSection(input: { return { section: input.section, items: page.counters, + storageBytesShadow: page.storageBytesShadow, truncated: page.truncated, nextStartAfter: page.nextStartAfter, pageSize, diff --git a/packages/worker/src/account/user-owned-surfaces.ts b/packages/worker/src/account/user-owned-surfaces.ts index 34d779fea8..92e3645f7b 100644 --- a/packages/worker/src/account/user-owned-surfaces.ts +++ b/packages/worker/src/account/user-owned-surfaces.ts @@ -130,7 +130,7 @@ export const accountUserOwnedDurableObjectSurfaces: ReadonlyArray + d1StorageBytes?: number }) { const stableUserId = testStableUserIdFromEmail(input.email) const dailyCounters = input.dailyCounters?.map((row) => ({ ...row })) ?? [] + const d1StorageBytes = input.d1StorageBytes ?? 0 return { stableUserId, db: { @@ -60,10 +62,12 @@ function createUsageTestDb(input: { ) return (row ? { count: row.count } : { count: 0 }) as T } + if (normalized.includes('d1_storage_bytes')) { + return { bytes: d1StorageBytes } as T + } if ( normalized.includes('count(*)') || - normalized.includes('sum(') || - normalized.includes('d1_storage_bytes') + normalized.includes('sum(') ) { return { count: 0, total: 0, bytes: 0 } as T } @@ -216,3 +220,24 @@ test('loadAccountUsageData returns plan rows and authoritative UserMeter daily c expect(currentFor(authoritative, 'concurrent_workflows')).toBe(3) expect(currentFor(authoritative, 'saved_packages')).toBe(4) }) + +test('loadAccountUsageData prefers D1 storage_bytes over a divergent UserMeter shadow', async () => { + const now = new Date('2026-07-25T12:00:00.000Z') + const email = 'usage-storage@example.com' + const userId = testStableUserIdFromEmail(email) + const { db } = createUsageTestDb({ + userId: 11, + email, + plan: 'pro', + packageCount: 0, + d1StorageBytes: 4_321, + }) + const usageEnv = withUsageEnv({ APP_DB: db }) + await usageEnv.meter.seedStorageBytes({ userId, bytes: 321 }) + const data = await loadAccountUsageData({ + env: usageEnv as Env, + userId: 11, + now, + }) + expect(currentFor(data, 'storage_bytes')).toBe(4_321) +}) From 3c13b53db7b5f79574cc3c9cc8cbdf1556e517fc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 07:16:36 +0000 Subject: [PATCH 15/15] fix(entitlements): make storage reserves fail closed Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 31 ++--- .../contributing/architecture/entitlements.md | 15 ++- .../worker/src/account/export.node.test.ts | 13 ++- packages/worker/src/account/export.ts | 5 +- .../worker/src/account/user-owned-surfaces.ts | 2 +- .../entitlements/entitlements.node.test.ts | 104 +++++++++++++++++ packages/worker/src/entitlements/service.ts | 110 ++++++++++++------ .../worker/src/entitlements/user-meter-do.ts | 12 +- .../entitlements/user-meter.workers.test.ts | 32 ++++- .../worker/src/test-support/user-meter.ts | 13 ++- 10 files changed, 273 insertions(+), 64 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 8e74c6138e..7bd1099f08 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -167,13 +167,14 @@ migration-safe chunked interface: StorageRunner `exportStorage({ pageSize, startAfter })` RPC as the dedicated storage export capability. User meter counters use `section: "user_meter"` and the `UserMeter.exportCounters` RPC (daily counters plus additive - `storageBytesShadow` when present; explicitly non-authoritative). R2 raw MIME, - attachment, avatar, and icon objects use `section: "r2_object"`; each response - contains at most one 256 KiB base64 chunk and an opaque cursor. Each request - uses bounded `LIMIT 1` ownership queries rather than reconstructing inventory. - Continuation cursors bind the source row, object key, size, and ETag; - ownership/key mutations and object overwrites are reported instead of mixing - generations. Missing objects are represented explicitly. + `storageBytesShadow` on the first page only when present; explicitly + non-authoritative). R2 raw MIME, attachment, avatar, and icon objects use + `section: "r2_object"`; each response contains at most one 256 KiB base64 + chunk and an opaque cursor. Each request uses bounded `LIMIT 1` ownership + queries rather than reconstructing inventory. Continuation cursors bind the + source row, object key, size, and ETag; ownership/key mutations and object + overwrites are reported instead of mixing generations. Missing objects are + represented explicitly. D1 manifest counts use bounded SQL `COUNT(*)` queries. D1 section rows are read with SQL-level keyset pagination: every query orders by the table's `rowid`, @@ -197,12 +198,13 @@ Durable Object export behavior: never pruned by retention. See [Run records](./run-records.md). - `UserMeter` exports daily entitlement counter rows through the `user_meter` section (`exportCounters` RPC; keyset pagination by UTC `day` and `resource`). - The same RPC may return additive `storageBytesShadow` when the schema-v4 - shadow row exists (`null` when never shadowed); section totals count it as one - row when present, but it is explicitly **non-authoritative** — usage and - enforcement read D1 `users.d1_storage_bytes`. Retention is self-enforced - inside the DO (seven UTC days of counter and inbound-delivery-claim rows); - shadow storage-byte state is not time-pruned. See + The same RPC may return additive `storageBytesShadow` on the first page only + (`startAfter` absent) when the schema-v4 shadow row exists (`null` on later + pages and when never shadowed); section totals count it as one row when + present, but it is explicitly **non-authoritative** — usage and enforcement + read D1 `users.d1_storage_bytes`. Retention is self-enforced inside the DO + (seven UTC days of counter and inbound-delivery-claim rows); shadow + storage-byte state is not time-pruned. See [Entitlements](./entitlements.md#usermeter-expand-phase). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. @@ -555,7 +557,8 @@ Account deletion calls `UserMeter.purge()` (one RPC per user, no D1 id scan; `deleteAll` clears counters, claims, and any shadow storage state). Account export pages `UserMeter.exportCounters` through the `user_meter` manifest section / `account_export_section` (daily counters plus additive -`storageBytesShadow` when present; shadow field is non-authoritative). +`storageBytesShadow` on the first page only when present; shadow field is +non-authoritative). ### Package state model diff --git a/docs/contributing/architecture/entitlements.md b/docs/contributing/architecture/entitlements.md index 2879002bd5..7171beca42 100644 --- a/docs/contributing/architecture/entitlements.md +++ b/docs/contributing/architecture/entitlements.md @@ -183,8 +183,11 @@ duplicate mirror SQL. D1 `users.d1_storage_bytes` and `users.d1_storage_bytes_updated_at` remain the **sole read, enforcement, and reconciliation authority** for the D1 payload component of `storage_bytes`. Every caller — including email — reserves through -the same atomic D1 `UPDATE` in `assertWithinStorageBytesEntitlement`; denial -semantics and `EntitlementLimitError` always come from D1. +the same conditional D1 `UPDATE` in `assertWithinStorageBytesEntitlement` +(bounded retry while a real users row remains under limit; fail closed if +contention never resolves; missing-user synthetic contexts keep prior +under-limit allow / over-limit deny). Denial semantics and +`EntitlementLimitError` always come from D1. UserMeter schema **v4** adds a singleton `storage_bytes_state` row (`id = 1`) as **best-effort shadow / future-cutover support only**. The DO row is never read @@ -211,9 +214,11 @@ failures are logged and never fail or poison the lane. Current `readUserD1StorageBytes` only. **Account export and purge:** `UserMeter.exportCounters` may return additive -non-authoritative `storageBytesShadow` when the shadow row exists (counted in -the `user_meter` section total). `UserMeter.purge()` clears counters, inbound -delivery claims, and any shadow storage state via `deleteAll`. +non-authoritative `storageBytesShadow` on the first page only (`startAfter` +absent) when the shadow row exists; subsequent pages return `null` so paged +consumers never double-count it (still counted once in the `user_meter` section +total). `UserMeter.purge()` clears counters, inbound delivery claims, and any +shadow storage state via `deleteAll`. ### Future storage authority flip (contract follow-up) diff --git a/packages/worker/src/account/export.node.test.ts b/packages/worker/src/account/export.node.test.ts index 97a962c81c..815d16f8cb 100644 --- a/packages/worker/src/account/export.node.test.ts +++ b/packages/worker/src/account/export.node.test.ts @@ -1588,9 +1588,11 @@ test('account export includes user_meter counters, pages them, and warns on trun : 0 const page = counters.slice(startIndex, startIndex + pageSize) const truncated = startIndex + pageSize < counters.length + const isFirstPage = + typeof input.startAfter !== 'string' || input.startAfter.length === 0 return { counters: page, - storageBytesShadow, + storageBytesShadow: isFirstPage ? storageBytesShadow : null, nextStartAfter: truncated ? `${page.at(-1)!.day}:${page.at(-1)!.resource}` : null, @@ -1647,10 +1649,15 @@ test('account export includes user_meter counters, pages them, and warns on trun startAfter: first.nextStartAfter ?? undefined, }) expect(second.items).toEqual(counters.slice(2)) - expect(second.storageBytesShadow).toEqual(storageBytesShadow) + expect(second.storageBytesShadow).toBeNull() expect(second.truncated).toBe(false) expect(second.nextStartAfter).toBeNull() - expect(exportCounters).toHaveBeenCalled() + expect(exportCounters).toHaveBeenCalledWith( + expect.objectContaining({ + pageSize: 2, + startAfter: first.nextStartAfter, + }), + ) exportCounters.mockImplementation(async () => ({ counters: [counters[0]!], diff --git a/packages/worker/src/account/export.ts b/packages/worker/src/account/export.ts index 297abc2c38..a2ac581228 100644 --- a/packages/worker/src/account/export.ts +++ b/packages/worker/src/account/export.ts @@ -283,7 +283,10 @@ export type AccountExportSectionResult = { nextStartAfter: string | null pageSize: number warnings: Array - /** Non-authoritative UserMeter storage-byte shadow; not part of paging. */ + /** + * Non-authoritative UserMeter storage-byte shadow. Present only on the + * first `user_meter` page (`startAfter` absent); later pages omit it. + */ storageBytesShadow?: UserMeterExportResult['storageBytesShadow'] } diff --git a/packages/worker/src/account/user-owned-surfaces.ts b/packages/worker/src/account/user-owned-surfaces.ts index 92e3645f7b..47a46d6fc3 100644 --- a/packages/worker/src/account/user-owned-surfaces.ts +++ b/packages/worker/src/account/user-owned-surfaces.ts @@ -130,7 +130,7 @@ export const accountUserOwnedDurableObjectSurfaces: ReadonlyArray { + const userId = await createStableUserIdFromEmail(plannedEmail) + const proLimit = planLimits.pro.maxStorageBytes + let reserveAttempts = 0 + let bytes = proLimit - 10 + const db = { + prepare(query: string) { + return { + bind(...params: Array) { + return { + async first() { + if ( + query.includes('SELECT plan, stripe_plan FROM users') || + query.includes('SELECT plan FROM users') + ) { + return { plan: 'pro', stripe_plan: null } as T + } + if (query.includes('SELECT d1_storage_bytes AS bytes')) { + return { bytes } as T + } + throw new Error(`Unsupported first query: ${query}`) + }, + async run() { + if ( + query.includes('SET d1_storage_bytes = d1_storage_bytes + ?') + ) { + reserveAttempts += 1 + if (reserveAttempts === 1) { + return { meta: { changes: 0 } } + } + const requested = Number(params[0]) + if (bytes + Number(params[3]) > Number(params[4])) { + return { meta: { changes: 0 } } + } + bytes += requested + return { meta: { changes: 1 } } + } + throw new Error(`Unsupported run query: ${query}`) + }, + } + }, + } + }, + } as unknown as D1Database + + await assertWithinStorageBytesEntitlement({ + db, + userId, + email: plannedEmail, + requested: 5, + }) + expect(reserveAttempts).toBe(2) + expect(bytes).toBe(proLimit - 5) +}) + +test('storage byte reserve fails closed when contention never resolves', async () => { + const userId = await createStableUserIdFromEmail(plannedEmail) + const proLimit = planLimits.pro.maxStorageBytes + let reserveAttempts = 0 + const db = { + prepare(query: string) { + return { + bind() { + return { + async first() { + if ( + query.includes('SELECT plan, stripe_plan FROM users') || + query.includes('SELECT plan FROM users') + ) { + return { plan: 'pro', stripe_plan: null } as T + } + if (query.includes('SELECT d1_storage_bytes AS bytes')) { + return { bytes: proLimit - 10 } as T + } + throw new Error(`Unsupported first query: ${query}`) + }, + async run() { + if ( + query.includes('SET d1_storage_bytes = d1_storage_bytes + ?') + ) { + reserveAttempts += 1 + return { meta: { changes: 0 } } + } + throw new Error(`Unsupported run query: ${query}`) + }, + } + }, + } + }, + } as unknown as D1Database + + await expect( + assertWithinStorageBytesEntitlement({ + db, + userId, + email: plannedEmail, + requested: 5, + }), + ).rejects.toThrow( + 'Storage byte reservation could not complete under concurrent updates.', + ) + expect(reserveAttempts).toBe(5) +}) + test('getPlanRank orders free < pro < partner < max', () => { expect(getPlanRank('free')).toBeLessThan(getPlanRank('pro')) expect(getPlanRank('pro')).toBeLessThan(getPlanRank('partner')) diff --git a/packages/worker/src/entitlements/service.ts b/packages/worker/src/entitlements/service.ts index ac3d13183b..5d3a5e6ae5 100644 --- a/packages/worker/src/entitlements/service.ts +++ b/packages/worker/src/entitlements/service.ts @@ -725,10 +725,10 @@ function scheduleUserMeterStorageBytesShadow(input: { void tracked } -export async function readUserD1StorageBytes(input: { +async function readUserD1StorageBytesRow(input: { db: D1Database userId: string -}): Promise { +}): Promise<{ bytes: number } | null> { const row = await input.db .prepare( `SELECT d1_storage_bytes AS bytes @@ -737,7 +737,16 @@ export async function readUserD1StorageBytes(input: { ) .bind(input.userId) .first<{ bytes: number }>() - return Math.max(0, Number(row?.bytes ?? 0)) + if (row == null) return null + return { bytes: Math.max(0, Number(row.bytes ?? 0)) } +} + +export async function readUserD1StorageBytes(input: { + db: D1Database + userId: string +}): Promise { + const row = await readUserD1StorageBytesRow(input) + return row?.bytes ?? 0 } /** @@ -985,11 +994,20 @@ export async function readCurrentEntitlementResourceUsage(input: { }) } +/** Bounded retries when a conditional D1 reserve loses to concurrent writers. */ +const storageBytesReserveMaxAttempts = 5 + /** * Atomically reserve D1 payload storage bytes against the plan limit. * Optional `env` schedules a best-effort UserMeter shadow after a successful * reserve (never awaited; failures logged only). `getCurrent` is check-only * and does not reserve in D1 or UserMeter. + * + * Real users never succeed unreserved: a failed conditional update re-reads + * row existence/current, throws {@link EntitlementLimitError} at the limit, + * retries while a real row remains under limit, and fails closed with an + * operational error if contention never resolves. Missing-user synthetic + * contexts keep their prior under-limit allow / over-limit deny semantics. */ export async function assertWithinStorageBytesEntitlement(input: { db: D1Database @@ -1020,42 +1038,66 @@ export async function assertWithinStorageBytesEntitlement(input: { }) const limit = resolvePlanLimit(plan, 'storage_bytes') const requested = Math.max(0, input.requested ?? 1) - const now = new Date().toISOString() - const result = await input.db - .prepare( - `UPDATE users - SET d1_storage_bytes = d1_storage_bytes + ?, - d1_storage_bytes_updated_at = ? - WHERE stable_user_id = ? - AND d1_storage_bytes + ? <= ?`, - ) - .bind(requested, now, input.userId, requested, limit) - .run() - if ((result.meta.changes ?? 0) > 0) { - scheduleUserMeterStorageBytesShadow({ + + for (let attempt = 0; attempt < storageBytesReserveMaxAttempts; attempt++) { + const now = new Date().toISOString() + const result = await input.db + .prepare( + `UPDATE users + SET d1_storage_bytes = d1_storage_bytes + ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ? + AND d1_storage_bytes + ? <= ?`, + ) + .bind(requested, now, input.userId, requested, limit) + .run() + if ((result.meta.changes ?? 0) > 0) { + scheduleUserMeterStorageBytesShadow({ + db: input.db, + env: input.env, + userId: input.userId, + waitUntil: input.waitUntil, + }) + return + } + + const row = await readUserD1StorageBytesRow({ db: input.db, - env: input.env, userId: input.userId, - waitUntil: input.waitUntil, }) - return + // Synthetic contexts have no users row to reserve against. Their plan + // has already failed closed to `free`; preserve that finite limit + // without inventing a user-owned counter row that reconciliation + // cannot resolve. + if (row == null) { + const current = 0 + if (current + requested <= limit) return + throw new EntitlementLimitError({ + resource: 'storage_bytes', + plan, + limit, + current, + upgradeHint: buildEntitlementUpgradeHint('storage_bytes'), + }) + } + + const current = row.bytes + if (current + requested > limit) { + throw new EntitlementLimitError({ + resource: 'storage_bytes', + plan, + limit, + current, + upgradeHint: buildEntitlementUpgradeHint('storage_bytes'), + }) + } + // Real row still under limit: another writer raced the conditional + // UPDATE. Retry the reserve rather than succeeding unreserved. } - const current = await readUserD1StorageBytes({ - db: input.db, - userId: input.userId, - }) - // Synthetic contexts have no users row to reserve against. Their plan has - // already failed closed to `free`; preserve that finite limit without - // inventing a user-owned counter row that reconciliation cannot resolve. - if (current + requested <= limit) return - throw new EntitlementLimitError({ - resource: 'storage_bytes', - plan, - limit, - current, - upgradeHint: buildEntitlementUpgradeHint('storage_bytes'), - }) + throw new Error( + 'Storage byte reservation could not complete under concurrent updates.', + ) } export type AssertWithinEntitlementInput = { diff --git a/packages/worker/src/entitlements/user-meter-do.ts b/packages/worker/src/entitlements/user-meter-do.ts index adc12de48d..8633bf0b91 100644 --- a/packages/worker/src/entitlements/user-meter-do.ts +++ b/packages/worker/src/entitlements/user-meter-do.ts @@ -112,7 +112,11 @@ export type UserMeterStorageBytesSetResult = UserMeterStorageBytesReadyState & { export type UserMeterExportResult = { counters: Array - /** Non-authoritative D1 storage-byte shadow; `null` when never synced. */ + /** + * Non-authoritative D1 storage-byte shadow. Emitted only on the first + * export page (`startAfter` absent); subsequent pages return `null` so + * paged consumers never double-count the singleton shadow. + */ storageBytesShadow: UserMeterStorageBytesState | null nextStartAfter: string | null truncated: boolean @@ -838,7 +842,11 @@ class UserMeterBase extends DurableObject { mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(revision), }) } - const storageRow = this.readStorageRow() + // Shadow is a singleton outside keyset paging — emit it once on the + // first page only so section totals and multi-page consumers do not + // double-count. + const includeStorageShadow = cursor == null + const storageRow = includeStorageShadow ? this.readStorageRow() : null const last = pageRows[pageRows.length - 1] return { counters, diff --git a/packages/worker/src/entitlements/user-meter.workers.test.ts b/packages/worker/src/entitlements/user-meter.workers.test.ts index 3722d4b156..5c1440355a 100644 --- a/packages/worker/src/entitlements/user-meter.workers.test.ts +++ b/packages/worker/src/entitlements/user-meter.workers.test.ts @@ -938,13 +938,41 @@ test('UserMeter storage RPCs, export shadow field, and purge work additively', a updatedAt: '2026-07-31T17:02:00.000Z', }) - const exported = await meter.exportCounters({}) - expect(exported.storageBytesShadow).toEqual({ + const day = '2026-07-31' + for (const resource of [ + 'email_receives_per_day', + 'email_sends_per_day', + 'execute_calls_per_day', + 'outbound_fetches_per_day', + ] as const) { + await meter.initialize({ + resource, + day, + count: 1, + updatedAt: '2026-07-31T17:03:00.000Z', + }) + } + + const firstPage = await meter.exportCounters({ pageSize: 2 }) + expect(firstPage.counters).toHaveLength(2) + expect(firstPage.truncated).toBe(true) + expect(firstPage.nextStartAfter).toEqual(expect.any(String)) + expect(firstPage.storageBytesShadow).toEqual({ bytes: 11, revision: 3, updatedAt: '2026-07-31T17:02:00.000Z', mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(3), }) + + const secondPage = await meter.exportCounters({ + pageSize: 2, + startAfter: firstPage.nextStartAfter, + }) + expect(secondPage.counters).toHaveLength(2) + expect(secondPage.truncated).toBe(false) + expect(secondPage.nextStartAfter).toBeNull() + expect(secondPage.storageBytesShadow).toBeNull() + await expect( readUserD1StorageBytes({ db: env.APP_DB, userId: user.userId }), ).resolves.toBe(0) diff --git a/packages/worker/src/test-support/user-meter.ts b/packages/worker/src/test-support/user-meter.ts index c02fe7f3b5..8452d2d284 100644 --- a/packages/worker/src/test-support/user-meter.ts +++ b/packages/worker/src/test-support/user-meter.ts @@ -182,7 +182,12 @@ export function createInMemoryUserMeterEnv() { storageByUser.delete(userId) return { ok: true as const } }, - async exportCounters() { + async exportCounters( + input: { + pageSize?: number + startAfter?: string | null + } = {}, + ) { const counters = [...rows.entries()].map(([entryKey, row]) => { const [resource, day] = entryKey.split('\0') return { @@ -194,7 +199,11 @@ export function createInMemoryUserMeterEnv() { mirrorUpdatedAt: userMeterMirrorUpdatedAtToken(row.revision), } }) - const storage = storageByUser.get(userId) + const includeStorageShadow = + typeof input.startAfter !== 'string' || input.startAfter.length === 0 + const storage = includeStorageShadow + ? storageByUser.get(userId) + : undefined const storageBytesShadow: UserMeterStorageBytesState | null = storage ? { bytes: storage.bytes,