From 8ef367ec3107d9e0deff6a2321b8b087313a05c8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 06:38:39 +0000 Subject: [PATCH 01/62] feat(email): scaffold per-user Mailbox durable object Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 164 +++- .../contributing/architecture/primitives.yaml | 9 + .../worker/src/account/export.node.test.ts | 155 ++++ packages/worker/src/account/export.ts | 89 ++ .../account/user-owned-surfaces.node.test.ts | 14 + .../worker/src/account/user-owned-surfaces.ts | 9 + .../src/app/account-deletion.node.test.ts | 41 + packages/worker/src/app/account-deletion.ts | 27 + packages/worker/src/email/mailbox-client.ts | 28 + packages/worker/src/email/mailbox-do.ts | 292 ++++++ .../src/email/mailbox-do.workers.test.ts | 660 ++++++++++++++ packages/worker/src/email/mailbox-export.ts | 293 ++++++ packages/worker/src/email/mailbox-mappers.ts | 319 +++++++ .../worker/src/email/mailbox-retention.ts | 137 +++ packages/worker/src/email/mailbox-schema.ts | 328 +++++++ packages/worker/src/email/mailbox-store.ts | 840 ++++++++++++++++++ packages/worker/src/email/mailbox-types.ts | 631 +++++++++++++ packages/worker/src/index.ts | 2 + ...er-scoped-durable-object-name.node.test.ts | 5 + .../src/user-scoped-durable-object-name.ts | 5 + packages/worker/worker-configuration.d.ts | 16 +- packages/worker/wrangler.jsonc | 16 + 22 files changed, 4051 insertions(+), 29 deletions(-) create mode 100644 packages/worker/src/email/mailbox-client.ts create mode 100644 packages/worker/src/email/mailbox-do.ts create mode 100644 packages/worker/src/email/mailbox-do.workers.test.ts create mode 100644 packages/worker/src/email/mailbox-export.ts create mode 100644 packages/worker/src/email/mailbox-mappers.ts create mode 100644 packages/worker/src/email/mailbox-retention.ts create mode 100644 packages/worker/src/email/mailbox-schema.ts create mode 100644 packages/worker/src/email/mailbox-store.ts create mode 100644 packages/worker/src/email/mailbox-types.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 7bd1099f08..c56f08bf91 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -78,22 +78,28 @@ Deletion must cover these user-owned surfaces: references a stale column. - **Durable Objects:** `JobManager`, `StorageRunner`, `RepoSession`, `RemoteConnectorSession`, `PackageRealtimeSession`, `PackageServiceInstance`, - `McpClientHub`, `RunLog`, and `UserMeter` are purged through account-deletion - RPCs after their D1 identifiers are collected (`RunLog` and `UserMeter` are - one object per user and need no D1 id scan). `MCP` objects remain SDK - session-keyed, while `mcp_agent_sessions` indexes each Durable Object id by - authenticated stable user id so account deletion can purge stored props, - conversation state, raw-fetch state, and transport storage before revoking - OAuth grants. + `McpClientHub`, `RunLog`, `UserMeter`, and `Mailbox` are purged through + account-deletion RPCs after their D1 identifiers are collected (`RunLog`, + `UserMeter`, and `Mailbox` are one object per user and need no D1 id scan). + During the Mailbox expand phase, `Mailbox.purge()` clears DO SQLite only; D1 + `email_*` rows and `EMAIL_BLOBS` deletion remain the authoritative account- + deletion path for mail content (see [Mailbox](#durable-objects-mailbox)). + `MCP` objects remain SDK session-keyed, while `mcp_agent_sessions` indexes + each Durable Object id by authenticated stable user id so account deletion can + purge stored props, conversation state, raw-fetch state, and transport storage + before revoking OAuth grants. - **Vectorize:** memory, job, and saved-package vector ids are derived from D1 rows and removed with `deleteByIds`. -- **R2:** raw email MIME blobs in `EMAIL_BLOBS` are enumerated from - `email_messages` (deterministic `emailRawMimeKey` keys) and attachment storage - keys while those rows still exist. A failed object delete aborts D1 - finalization so those inventory rows remain available for retry. Rows owned by - `system:email` keep their blobs here (they are not user data); those blobs are - removed when the system-email retention prune deletes the messages through the - shared delete-message helper. +- **R2:** raw email MIME and attachment blobs in `EMAIL_BLOBS` are deleted by + per-user prefix cleanup (`email-raw:v1:{userId}/` and + `email-attachment:v1:{userId}/`) plus any remaining D1-inventoried keys + outside those prefixes. A failed object delete aborts D1 finalization so + inventory rows remain available for retry. Rows owned by `system:email` keep + their blobs here (they are not user data); those blobs are removed when the + system-email retention prune deletes the messages through the shared + delete-message helper. Mailbox expand-phase `purge` does not delete R2 objects + — D1 email rows and this existing R2 prefix deletion stay authoritative during + expand. - **KV:** published bundle artifact keys, source/manifest snapshot keys, community listing snapshots, and per-user package retriever cache/index keys in `BUNDLE_ARTIFACTS_KV` are deleted before D1 projection rows are removed. @@ -168,8 +174,9 @@ migration-safe chunked interface: storage export capability. User meter counters use `section: "user_meter"` and the `UserMeter.exportCounters` RPC (daily counters plus additive `storageBytesShadow` on the first page only when present; explicitly - non-authoritative). R2 raw MIME, attachment, avatar, and icon objects use - `section: "r2_object"`; each response contains at most one 256 KiB base64 + non-authoritative). Mailbox metadata uses `section: "mailbox"` and the + `Mailbox.exportMailbox` RPC. R2 raw MIME, attachment, avatar, and icon objects + use `section: "r2_object"`; each response contains at most one 256 KiB base64 chunk and an opaque cursor. Each request uses bounded `LIMIT 1` ownership queries rather than reconstructing inventory. Continuation cursors bind the source row, object key, size, and ETag; ownership/key mutations and object @@ -206,6 +213,13 @@ Durable Object export behavior: (seven UTC days of counter and inbound-delivery-claim rows); shadow storage-byte state is not time-pruned. See [Entitlements](./entitlements.md#usermeter-expand-phase). +- `Mailbox` exports per-user email metadata (threads, messages, attachments, + delivery events) through the account-export `mailbox` section (`exportMailbox` + RPC; keyset pagination with prefixed cursors over those tables). Manifest + counts use `countMailbox`. Phase 1 registers this consumption without + dual-write or a change of D1 authority — D1 `email_*` rows remain the live + source of truth and are still exported in the `d1` section. See + [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -560,6 +574,110 @@ section / `account_export_section` (daily counters plus additive `storageBytesShadow` on the first page only when present; shadow field is non-authoritative). +## Durable Objects (`Mailbox`) + +User-owned email **metadata** moves to a per-user `Mailbox` Durable Object with +SQLite (`packages/worker/src/email/mailbox-do.ts` and siblings under +`mailbox-*.ts`; client in `packages/worker/src/email/mailbox-client.ts`). The +Worker binding is `MAILBOX` (class `Mailbox`; Wrangler SQLite migration tag +`v22` via `new_sqlite_classes` in `packages/worker/wrangler.jsonc`). Raw MIME +and outbound attachment bytes stay in `EMAIL_BLOBS` R2; the DO stores object +keys, not payload bytes. + +Naming matches `RunLog`, `JobManager`, and `UserMeter`: one object per untrimmed +stable MCP `userId` via `mailboxDurableObjectName(userId)` → +`idFromName(userId)` in +`packages/worker/src/user-scoped-durable-object-name.ts`. There is no `user_id` +column inside the DO because the object identity is the user. + +**SQLite ownership** (schema version in `mailbox_meta`, currently +`mailboxSchemaVersion = 2`): + +- `email_threads`, `email_messages`, `email_attachments`, and + `email_delivery_events` for the owning user (same logical names as the D1 + tables they will eventually replace) +- latest per-message delivery status on `email_messages.delivery_status`, kept + separate from send-request `processing_status` + +**Retention** is self-enforced inside the DO with alarms +(`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). +Alarm-driven deletes are strict blob-before-row for `EMAIL_BLOBS` keys (failed +blob deletes skip the row for retry), matching today's D1 retention ordering. + +Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; +result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state +only and reinitializes schema — it does **not** delete R2 objects. D1 `email_*` +row deletion and existing `EMAIL_BLOBS` prefix deletion remain authoritative for +mail content. Account export pages Mailbox state through the `mailbox` section +(`exportMailbox` / `countMailbox`). + +### Expand/contract phases + +This is an expand/contract migration. **The current PR is phase 1 only:** +additive scaffold (DO class, binding, naming, client, mirror/export/purge RPCs, +and alarm retention) **plus registered account-deletion `purge` and +account-export `mailbox` section consumption**. It does **not** dual-write, does +**not** change live email read/write authority, and leaves D1 `email_*` tables +authoritative for all live paths. D1 email rows and existing R2 inventory +deletion stay authoritative during expand. + +1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze + `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / + `exportMailbox` / `countMailbox` / `purge` RPCs and alarm retention, and + register account deletion/export consumption. No dual-write; D1 remains sole + authority for live mail. +2. **D1-authoritative dual-write + parity** — every user-mail mutation that + writes D1 also writes the DO; parity counters and reconciliation prove DO + completeness against D1. D1 stays authoritative for reads. +3. **Reads cut over after production parity** — user-mail reads move to the DO + only after production parity is verified. D1 writes continue. +4. **D1 write-off / event retirement** — stop writing moved user-mail metadata + to D1; retire dual-write and event/mirror machinery used only for the + migration. +5. **Later contract migrations** — drop retired D1 user-mail tables/columns only + after verification. No premature schema deletion. + +A cron sibling track owns scheduled-lane wiring (for example parity / +reconciliation ticks). This Mailbox track owns DO storage semantics, alarm +retention, and the inbound durability boundary below. + +### What stays in D1 + +- **`system:email` operator inbox** — remains in D1 for cross-account / admin + access, fixed bounded caps, and separate system-email retention. Those rows + stay excluded from account deletion and export + (`accountUserDataExcludedOwnerIds`). They are not migrated into per-user + Mailbox objects. +- **Low-write email config** — sender identities, inboxes, inbox addresses, + sender rules, and similar low-churn configuration stay in D1. +- **Provider-message reverse lookup** — outbound `provider_message_id` → owner + resolution stays in D1 until the delivery webhook path already knows the + owning user. Contextless provider-id reverse lookups must not require + enumerating per-user Mailbox objects. + +### Inbound durability boundary (eventual DO phase) + +Today's D1-authoritative inbound commit boundary is documented under +[R2 (`EMAIL_BLOBS`)](#r2-community_assets-email_blobs): thread prework + R2 put + +- D1 message/attachment rows are pre-commit; `touchEmailThread` / `received` + delivery-event writes are post-commit best-effort; ambiguous attachment-insert + failures acknowledge rather than risk duplicates. + +When Mailbox becomes authoritative, the same shape applies with the DO as the +metadata store: + +- **Pre-commit:** thread prework + R2 raw-MIME put + atomic Mailbox + message/attachment commit. +- **Post-commit (best effort):** thread touch and delivery-event writes — log + failures without throwing (retry would duplicate mail). +- **Ambiguity:** if attachment commit fails but message cleanup (or a residual + probe) cannot prove the pre-commit state, acknowledge the already-created + message (logged, non-retry) rather than risking a duplicate on Email Routing + retry. + +Phase 1 scaffolding does not change that live D1 boundary. + ### Package state model Saved packages are the only top-level persisted primitive. Their state maps onto @@ -599,6 +717,9 @@ via `durableObjectNameFromParts`); domain helpers such as daily-entitlement meter DO per user (untrimmed stable id, same as `RunLog`), plus optional schema-v4 D1 storage-byte shadow. See [Entitlements](./entitlements.md#usermeter-expand-phase). +- `Mailbox` — `mailboxDurableObjectName(userId)` → `idFromName(userId)`. One + email-metadata DO per user (untrimmed stable id, same as `RunLog`). See + [Mailbox](#durable-objects-mailbox). - `McpClientHub` — `mcpClientHubDurableObjectName(userId)` → `idFromName(userId.trim())`. - `StorageRunner` — `storageRunnerDurableObjectName(userId, storageId)` → @@ -666,6 +787,9 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` [Run records](./run-records.md)) - `USER_METER` (Durable Objects; per-user daily entitlement counters — see [Entitlements](./entitlements.md#usermeter-expand-phase)) +- `MAILBOX` (Durable Objects; per-user email metadata — see + [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export + consumption without changing D1 authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) @@ -914,6 +1038,7 @@ to `durableObjectNameFromParts`). - `RunLog`: `idFromName(userId)` (no trim); one execution-history DO per user. - `UserMeter`: `idFromName(userId)` (no trim); one daily-entitlement meter DO per user, plus optional schema-v4 D1 storage-byte shadow. +- `Mailbox`: `idFromName(userId)` (no trim); one email-metadata DO per user. - `McpClientHub`: `idFromName(userId.trim())`. - `StorageRunner`: `idFromName(JSON.stringify([userId, storageId]))`. - `RepoSession`: `idFromName(repo_sessions.id)`; the key is not user-prefixed, @@ -1091,7 +1216,9 @@ Current retention policies: which prunes messages (and their `EMAIL_BLOBS` raw-MIME blobs) and delivery events older than 90 days in bounded batches within its own time budget, deletes stale `system_email_daily_counters`, and caps stored system messages - at 5000. + at 5000. After Mailbox cut-over, user-owned delivery-event retention moves to + the per-user Mailbox DO alarm (still 90 days, strict blob-before-row); D1 + policies here remain authoritative until that phase. - `email_messages` / `email_attachments` / `email_threads`: user-owned messages (excluding the `system:email` owner) keep 365 days, deleted oldest first in batches. Retention deletes the deterministic @@ -1099,6 +1226,9 @@ Current retention policies: blob delete fails, those rows are skipped and still selected on later runs so cleanup can retry. Dependent `email_attachments` rows are deleted before their messages, and threads left with no messages are pruned for the affected users. + After Mailbox cut-over, the same 365-day window and blob-before-row ordering + are self-enforced by the Mailbox DO alarm; `system:email` stays on the D1 + system-email retention job. - `entitlement_daily_counters`: expand-phase **mirror** of UserMeter daily counters (authoritative state lives in the per-user `UserMeter` DO). Rows keep 400 days by `day` key until mirror retirement is verified after diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index 5302603cd6..6a0731c858 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -522,6 +522,15 @@ primitives: - docs/contributing/architecture/data-storage.md - docs/contributing/architecture/entitlements.md + - id: mailbox + group: storage + name: Mailbox + summary: Per-user Mailbox DO SQLite for email metadata; R2 keeps raw MIME. + code: + - packages/worker/src/email/mailbox- + docs: + - docs/contributing/architecture/data-storage.md + - id: email-blobs-r2 group: storage name: Email blob storage (R2) diff --git a/packages/worker/src/account/export.node.test.ts b/packages/worker/src/account/export.node.test.ts index 815d16f8cb..657b5a1f42 100644 --- a/packages/worker/src/account/export.node.test.ts +++ b/packages/worker/src/account/export.node.test.ts @@ -1677,6 +1677,161 @@ test('account export includes user_meter counters, pages them, and warns on trun ) }) +test('account export includes mailbox rows, pages them, and warns on truncation', async () => { + const { sqlite, db } = createMigratedDb() + sqlite.exec(` + INSERT INTO users ( + id, username, email, password_hash, created_at, updated_at, + email_verified_at, stable_user_id + ) + VALUES ( + 1, 'user-a', 'a@example.com', 'password-hash-a', '2026-07-05', + '2026-07-05', '2026-07-05', 'user-aaa' + ); + `) + const rows = [ + { + kind: 'thread' as const, + row: { + id: 'thread-1', + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: null, + lastMessageAt: '2026-07-30T00:00:00.000Z', + createdAt: '2026-07-30T00:00:00.000Z', + updatedAt: '2026-07-30T00:00:00.000Z', + }, + }, + { + kind: 'message' as const, + row: { + id: 'message-1', + threadId: 'thread-1', + inboxId: 'inbox-1', + direction: 'inbound' as const, + processingStatus: 'received' as const, + deliveryStatus: 'delivered' as const, + classification: 'personal' as const, + subject: 'hello', + fromAddress: 'a@example.com', + toAddresses: ['b@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + messageIdHeader: null, + inReplyToHeader: null, + referencesHeader: null, + sentAt: '2026-07-30T00:00:00.000Z', + receivedAt: '2026-07-30T00:00:00.000Z', + rawMimeKey: null, + rawMimeStorageKind: 'unavailable' as const, + bodyText: 'hi', + bodyHtml: null, + snippet: 'hi', + hasAttachments: false, + createdAt: '2026-07-30T00:00:00.000Z', + updatedAt: '2026-07-30T00:00:00.000Z', + }, + }, + { + kind: 'attachment' as const, + row: { + id: 'attachment-1', + messageId: 'message-1', + filename: 'file.txt', + contentType: 'text/plain', + sizeBytes: 3, + storageKey: null, + storageKind: 'unavailable' as const, + contentId: null, + isInline: false, + createdAt: '2026-07-30T00:00:00.000Z', + }, + }, + ] + const exportMailbox = vi.fn( + async (input: { pageSize?: number; startAfter?: string | null }) => { + const pageSize = input.pageSize ?? 100 + const startIndex = input.startAfter + ? rows.findIndex((row) => row.row.id === input.startAfter) + 1 + : 0 + const page = rows.slice(startIndex, startIndex + pageSize) + const truncated = startIndex + pageSize < rows.length + return { + rows: page, + nextStartAfter: truncated ? page.at(-1)!.row.id : null, + truncated, + } + }, + ) + const countMailbox = vi.fn(async () => ({ + threads: 1, + messages: 1, + attachments: 1, + deliveryEvents: 0, + })) + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const env = { + APP_DB: db, + MAILBOX: { + idFromName, + get: () => ({ exportMailbox, countMailbox }), + }, + } as unknown as Env + + const accountExport = await createAccountExport({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + }) + expect(idFromName).toHaveBeenCalledWith('user-aaa') + expect(accountExport.manifest.sections.mailbox?.count).toBe(3) + expect(accountExport.durableObjects.mailbox).toEqual({ + rows, + nextStartAfter: null, + truncated: false, + }) + + const first = await readAccountExportSection({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + section: 'mailbox', + pageSize: 2, + }) + expect(first.items).toEqual(rows.slice(0, 2)) + expect(first.truncated).toBe(true) + expect(first.nextStartAfter).toBe('message-1') + + const second = await readAccountExportSection({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + section: 'mailbox', + pageSize: 2, + startAfter: first.nextStartAfter ?? undefined, + }) + expect(second.items).toEqual(rows.slice(2)) + expect(second.truncated).toBe(false) + expect(second.nextStartAfter).toBeNull() + expect(exportMailbox).toHaveBeenCalled() + + exportMailbox.mockImplementation(async () => ({ + rows: [rows[0]!], + nextStartAfter: 'cursor-more', + truncated: true, + })) + const truncatedExport = await createAccountExport({ + env, + dbUserId: 1, + mcpUserId: 'user-aaa', + }) + expect(truncatedExport.durableObjects.mailbox?.truncated).toBe(true) + expect(truncatedExport.manifest.warnings).toContain( + 'Mailbox rows were truncated in the full export; use account_export_section with section "mailbox" to retrieve additional pages.', + ) +}) + test('run_records section paging preserves exportRuns cursor across dedicated phases', async () => { const { sqlite, db } = createMigratedDb() sqlite.exec(` diff --git a/packages/worker/src/account/export.ts b/packages/worker/src/account/export.ts index a2ac581228..a5796c6378 100644 --- a/packages/worker/src/account/export.ts +++ b/packages/worker/src/account/export.ts @@ -39,6 +39,8 @@ import { userMeterRpc, } from '#worker/entitlements/user-meter-client.ts' import { type UserMeterExportResult } from '#worker/entitlements/user-meter-do.ts' +import { mailboxNamespace, mailboxRpc } from '#worker/email/mailbox-client.ts' +import { type MailboxExportResult } from '#worker/email/mailbox-types.ts' import { resolveUserStableId } from '#worker/user-id.ts' import { listAccountUserPackageServices, @@ -61,6 +63,7 @@ export const accountExportSectionNames = [ 'job_manager', 'run_records', 'user_meter', + 'mailbox', 'remote_connector_session', 'package_service', 'oauth_grants', @@ -133,6 +136,7 @@ type ManifestInventoryCounts = { storageRunners: number runRecords: number userMeterCounters: number + mailboxRows: number remoteConnectorSessions: number packageServices: number artifactRepos: number @@ -250,6 +254,7 @@ type AccountExportDurableObjects = { jobManager: unknown | null runRecords: RunRecordsExportPayload | null userMeter: UserMeterExportResult | null + mailbox: MailboxExportResult | null remoteConnectorSessions: Array<{ instanceId: string export: RemoteConnectorSessionExport | null @@ -1035,6 +1040,7 @@ async function collectManifestInventoryCounts(input: { storageRunners, runRecords, userMeterCounters, + mailboxRows, remoteConnectorSessions, packageServices, artifactRepos, @@ -1061,6 +1067,19 @@ async function collectManifestInventoryCounts(input: { }) return countUserMeterExportEntries(page) }), + safeCount('mailbox rows', async () => { + if (!mailboxNamespace(input.env)) return 0 + const counts = await mailboxRpc({ + env: input.env, + userId: input.userId, + }).countMailbox() + return ( + counts.threads + + counts.messages + + counts.attachments + + counts.deliveryEvents + ) + }), safeCount('remote connectors', async () => countScalar( input.env, @@ -1099,6 +1118,7 @@ async function collectManifestInventoryCounts(input: { storageRunners, runRecords, userMeterCounters, + mailboxRows, remoteConnectorSessions, packageServices, artifactRepos, @@ -1371,6 +1391,36 @@ async function exportUserMeterCounters(input: { } } +async function exportMailboxRows(input: { + env: AccountExportEnv + userId: string + warnings: Array +}): Promise { + try { + if (!mailboxNamespace(input.env)) { + input.warnings.push( + 'MAILBOX binding was unavailable; mailbox rows were not exported.', + ) + return null + } + const page = await mailboxRpc({ + env: input.env, + userId: input.userId, + }).exportMailbox({ + pageSize: maxExportPageSize, + }) + if (page.truncated) { + input.warnings.push( + `Mailbox rows were truncated in the full export; use account_export_section with section "mailbox" to retrieve additional pages.`, + ) + } + return page + } catch (error) { + input.warnings.push(`Mailbox export failed: ${getErrorMessage(error)}`) + return null + } +} + async function exportRemoteConnectorSessions(input: { env: AccountExportEnv userId: string @@ -1469,6 +1519,7 @@ async function exportDurableObjects(input: { packageServices, runRecords, userMeter, + mailbox, ] = await Promise.all([ exportStorageRunners({ env: input.env, @@ -1498,6 +1549,11 @@ async function exportDurableObjects(input: { userId: input.userId, warnings: input.warnings, }), + exportMailboxRows({ + env: input.env, + userId: input.userId, + warnings: input.warnings, + }), ]) let jobManager: unknown | null = null try { @@ -1512,6 +1568,7 @@ async function exportDurableObjects(input: { jobManager, runRecords, userMeter, + mailbox, remoteConnectorSessions, packageServices, storageRunners, @@ -1587,6 +1644,17 @@ function buildManifest(input: { ), discovery: { section: 'user_meter' }, } + sections.mailbox = { + count: + input.durableObjects?.mailbox == null + ? (input.inventoryCounts?.mailboxRows ?? 0) + : input.durableObjects.mailbox.rows.length, + warnings: input.warnings.filter( + (warning) => + warning.startsWith('Mailbox ') || warning.startsWith('MAILBOX '), + ), + discovery: { section: 'mailbox' }, + } sections.remote_connector_sessions = { count: input.inventoryCounts?.remoteConnectorSessions ?? @@ -1927,6 +1995,27 @@ export async function readAccountExportSection(input: { warnings, } } + if (input.section === 'mailbox') { + if (!mailboxNamespace(input.env)) { + throw new Error('MAILBOX binding was unavailable.') + } + const pageSize = normalizePageSize(input.pageSize) + const page = await mailboxRpc({ + env: input.env, + userId: input.mcpUserId, + }).exportMailbox({ + pageSize, + startAfter: input.startAfter ?? null, + }) + return { + section: input.section, + items: page.rows, + truncated: page.truncated, + nextStartAfter: page.nextStartAfter, + pageSize, + warnings, + } + } if (input.section === 'remote_connector_session') { if (!input.instanceId) { throw new Error( diff --git a/packages/worker/src/account/user-owned-surfaces.node.test.ts b/packages/worker/src/account/user-owned-surfaces.node.test.ts index 41cb3b0280..2882e4311a 100644 --- a/packages/worker/src/account/user-owned-surfaces.node.test.ts +++ b/packages/worker/src/account/user-owned-surfaces.node.test.ts @@ -60,6 +60,7 @@ test('account deletion and export consume the out-of-band surface registry', () expect(coverage.r2SurfaceIds.has('email_raw_mime')).toBe(true) expect(coverage.r2SurfaceIds.has('user_avatar')).toBe(true) expect(coverage.durableObjectIds.has('user_meter')).toBe(true) + expect(coverage.durableObjectIds.has('mailbox')).toBe(true) expect( accountUserOwnedDurableObjectSurfaces.find( (surface) => surface.id === 'user_meter', @@ -69,6 +70,15 @@ test('account deletion and export consume the out-of-band surface registry', () deletionResultKey: 'userMeters', export: 'include', }) + expect( + accountUserOwnedDurableObjectSurfaces.find( + (surface) => surface.id === 'mailbox', + ), + ).toMatchObject({ + binding: 'MAILBOX', + deletionResultKey: 'mailboxes', + export: 'include', + }) expect( accountUserOwnedKvKeySchemes.some((scheme) => scheme.prefixTemplate?.includes('source-snapshot:v1:'), @@ -76,8 +86,12 @@ test('account deletion and export consume the out-of-band surface registry', () ).toBe(true) expect(accountExportSource).toContain("'user_meter'") expect(accountExportSource).toContain('userMeterRpc') + expect(accountExportSource).toContain("'mailbox'") + expect(accountExportSource).toContain('mailboxRpc') expect(accountDeletionSource).toContain('userMeterRpc') expect(accountDeletionSource).toContain('userMeters') + expect(accountDeletionSource).toContain('mailboxRpc') + expect(accountDeletionSource).toContain('mailboxes') expect(accountDeletionSource).toContain( "from '#worker/account/user-owned-surfaces.ts'", diff --git a/packages/worker/src/account/user-owned-surfaces.ts b/packages/worker/src/account/user-owned-surfaces.ts index 47a46d6fc3..885110edd9 100644 --- a/packages/worker/src/account/user-owned-surfaces.ts +++ b/packages/worker/src/account/user-owned-surfaces.ts @@ -9,6 +9,7 @@ export type UserOwnedDurableObjectSurface = { | 'package_service_instance' | 'run_log' | 'user_meter' + | 'mailbox' | 'mcp' binding: string /** Result key used in AccountDeletionResult.clearedDurableObjects when purged */ @@ -132,6 +133,14 @@ export const accountUserOwnedDurableObjectSurfaces: ReadonlyArray ({ purge: async () => ({ ok: true as const }) }), }, + MAILBOX: { + idFromName: durableObjectId, + get: () => ({ purge: async () => ({ ok: true as const }) }), + }, JOB_MANAGER: { idFromName: durableObjectId, get: () => ({ purgeUser: async () => ({ ok: true as const }) }), @@ -1111,6 +1115,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async const clearStorageMock = vi.fn(async () => ({ ok: true as const })) const clearRunLogMock = vi.fn(async () => ({ ok: true as const })) const purgeUserMeterMock = vi.fn(async () => ({ ok: true as const })) + const purgeMailboxMock = vi.fn(async () => ({ ok: true as const })) const purgeJobManagerMock = vi.fn(async () => ({ ok: true as const })) const purgeRepoSessionMock = vi.fn(async () => ({ ok: true as const })) const purgeRemoteConnectorMock = vi.fn(async () => ({ ok: true as const })) @@ -1140,6 +1145,10 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async idFromName: (name: string) => name as unknown as DurableObjectId, get: () => ({ purge: purgeUserMeterMock }), }, + MAILBOX: { + idFromName: (name: string) => name as unknown as DurableObjectId, + get: () => ({ purge: purgeMailboxMock }), + }, JOB_MANAGER: { idFromName: (name: string) => name as unknown as DurableObjectId, get: () => ({ purgeUser: purgeJobManagerMock }), @@ -1441,6 +1450,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async storageRunners: 6, runLogs: 1, userMeters: 1, + mailboxes: 1, jobManagers: 1, repoSessions: 1, remoteConnectorSessions: 1, @@ -1454,6 +1464,7 @@ test('deleteUserAccount cascades user-scoped rows for the requested user', async }) expect(clearRunLogMock).toHaveBeenCalledTimes(1) expect(purgeUserMeterMock).toHaveBeenCalledTimes(1) + expect(purgeMailboxMock).toHaveBeenCalledTimes(1) expect(purgeMcpClientHubMock).toHaveBeenCalledTimes(1) expect(purgeMcpAgentSessionMock).toHaveBeenCalledWith({ userId: userAaa, @@ -2031,6 +2042,36 @@ test('account deletion reports a missing USER_METER binding and remains retryabl ]) }) +test('account deletion reports a missing MAILBOX binding and remains retryable', async () => { + const { db, rows } = createTestDb({ + users: [{ id: 1, email: 'a@example.com', stable_user_id: 'user-aaa' }], + }) + await expect( + deleteUserAccount({ + env: createSuccessfulDeletionEnv(db, { + MAILBOX: undefined, + }), + dbUserId: 1, + mcpUserId: 'user-aaa', + }), + ).rejects.toMatchObject({ + cleanupErrors: expect.arrayContaining([ + 'MAILBOX binding was unavailable; the mailbox Durable Object was not purged.', + ]), + partialResult: expect.objectContaining({ + clearedDurableObjects: expect.objectContaining({ + mailboxes: 0, + }), + }), + }) + expect(rows.users).toEqual([ + expect.objectContaining({ + id: 1, + deleting_at: expect.any(String), + }), + ]) +}) + test('account deletion purges a StorageRunner known only via user_storage_buckets', async () => { const userId = 'user-bucket-only' const clearStorage = vi.fn(async () => ({ ok: true as const })) diff --git a/packages/worker/src/app/account-deletion.ts b/packages/worker/src/app/account-deletion.ts index bb2d825d7d..c55d5d0992 100644 --- a/packages/worker/src/app/account-deletion.ts +++ b/packages/worker/src/app/account-deletion.ts @@ -21,6 +21,7 @@ import { userMeterNamespace, userMeterRpc, } from '#worker/entitlements/user-meter-client.ts' +import { mailboxNamespace, mailboxRpc } from '#worker/email/mailbox-client.ts' import { listAccountUserPackageServices, listAccountUserStorageIds, @@ -621,6 +622,27 @@ async function purgeUserMeter(input: { } } +async function purgeMailbox(input: { + env: Env + userId: string + warnings: Array +}): Promise { + try { + if (!mailboxNamespace(input.env)) { + input.warnings.push( + 'MAILBOX binding was unavailable; the mailbox Durable Object was not purged.', + ) + return 0 + } + await mailboxRpc({ env: input.env, userId: input.userId }).purge() + return 1 + } catch (error) { + const message = getErrorMessage(error) + input.warnings.push(`Mailbox purge failed: ${message}`) + return 0 + } +} + async function purgeJobManager(input: { env: Env userId: string @@ -1122,6 +1144,11 @@ export async function deleteUserAccount(input: { userId: input.mcpUserId, warnings, }) + result.clearedDurableObjects.mailboxes = await purgeMailbox({ + env: input.env, + userId: input.mcpUserId, + warnings, + }) if (input.env.BUNDLE_ARTIFACTS_KV) { const sourceSnapshotKeys = await listKvKeysByPrefix({ diff --git a/packages/worker/src/email/mailbox-client.ts b/packages/worker/src/email/mailbox-client.ts new file mode 100644 index 0000000000..a261765b0d --- /dev/null +++ b/packages/worker/src/email/mailbox-client.ts @@ -0,0 +1,28 @@ +import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { type MailboxRpc } from './mailbox-do.ts' + +export type MailboxEnv = { + MAILBOX?: DurableObjectNamespace +} + +export function mailboxNamespace( + env: MailboxEnv, +): DurableObjectNamespace | null { + return env.MAILBOX ?? null +} + +/** Typed per-user Mailbox RPC stub; throws when `MAILBOX` is missing. */ +export function mailboxRpc(input: { + env: MailboxEnv + userId: string +}): MailboxRpc { + const namespace = mailboxNamespace(input.env) + if (!namespace) { + throw new Error('MAILBOX Durable Object binding is not configured.') + } + return namespace.get( + namespace.idFromName(mailboxDurableObjectName(input.userId)), + ) as unknown as MailboxRpc +} + +export type { MailboxRpc } diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts new file mode 100644 index 0000000000..cce457aca3 --- /dev/null +++ b/packages/worker/src/email/mailbox-do.ts @@ -0,0 +1,292 @@ +import * as Sentry from '@sentry/cloudflare' +import { DurableObject } from 'cloudflare:workers' +import { buildSentryOptions } from '#worker/sentry-options.ts' +import { + type EmailDeliveryEventType, + type EmailDeliveryStatus, +} from './types.ts' +import { + enforceMailboxRetention, + mailboxRetentionAlarmAtMs, + nextMailboxRetentionDueAtMs, +} from './mailbox-retention.ts' +import { MailboxStore } from './mailbox-store.ts' +import { + assertMailboxNonEmptyString, + mailboxRetentionAlarmSkewMs, + type MailboxAttachmentInput, + type MailboxAttachmentRecord, + type MailboxBlobReferencePage, + type MailboxCountResult, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, + type MailboxExportResult, + type MailboxListMessagesInput, + type MailboxMessageInput, + type MailboxMessageRecord, + type MailboxSearchMessagesInput, + type MailboxThreadInput, + type MailboxThreadRecord, +} from './mailbox-types.ts' + +/** + * Per-owner Mailbox Durable Object: SQLite metadata for threads, messages, + * attachments, and delivery events. Object identity is ownership — no + * `user_id` columns. Raw MIME / external attachment bytes stay in + * `EMAIL_BLOBS`; rows retain keys. `system:email` stays in D1 by design. + * + * Scaffold only: dual-write / cutover callers are not wired here. + */ + +class MailboxBase extends DurableObject { + private readonly store: MailboxStore + /** + * In-isolate cache: once an alarm is scheduled, hot writes skip + * getAlarm/setAlarm. Cleared when retention becomes idle or a write may + * need an earlier wake. + */ + private retentionAlarmArmed = false + /** + * In-isolate: no future retention work. Cleared on every data write so + * ensureRetentionAlarm re-arms for the new row's due-time. + */ + private retentionIdleConfirmed = false + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env) + this.store = new MailboxStore(ctx.storage) + this.ctx.blockConcurrencyWhile(async () => { + this.store.initializeSchema() + // Observe existing alarm only — never arm in the constructor. + this.retentionAlarmArmed = (await this.ctx.storage.getAlarm()) != null + }) + } + + /** + * Arming rule (keep this comment accurate — reviewers rely on it): + * + * - Schedule at most one alarm for the soonest retention due-time. + * - Overdue work uses an hourly retry delay (not now+1s) so persistent R2 + * failures cannot storm alarm wakes; future rows keep due-time scheduling. + * - `retentionAlarmArmed` skips getAlarm/setAlarm on hot writes once armed. + * - `retentionIdleConfirmed` skips re-evaluation only while no new row has + * been written; every write clears it so the next ensure re-arms. + * - Never arm in the constructor. + */ + private async ensureRetentionAlarm() { + if (this.retentionAlarmArmed) return + if (this.retentionIdleConfirmed) return + + const dueAtMs = nextMailboxRetentionDueAtMs(this.store) + const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) + if (alarmAt == null) { + this.retentionIdleConfirmed = true + this.retentionAlarmArmed = false + return + } + + this.retentionIdleConfirmed = false + const existing = await this.ctx.storage.getAlarm() + if ( + existing != null && + Math.abs(existing - alarmAt) < mailboxRetentionAlarmSkewMs + ) { + this.retentionAlarmArmed = true + return + } + await this.ctx.storage.setAlarm(alarmAt) + this.retentionAlarmArmed = true + } + + private markRetentionDirty() { + this.retentionIdleConfirmed = false + this.retentionAlarmArmed = false + } + + async alarm(): Promise { + await enforceMailboxRetention({ + store: this.store, + blobs: this.env.EMAIL_BLOBS, + }) + const dueAtMs = nextMailboxRetentionDueAtMs(this.store) + const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) + if (alarmAt == null) { + this.retentionAlarmArmed = false + this.retentionIdleConfirmed = true + return + } + await this.ctx.storage.setAlarm(alarmAt) + this.retentionAlarmArmed = true + this.retentionIdleConfirmed = false + } + + /** Atomic mirror of thread + message + attachments for dual-write. */ + async mirrorMessage(input: { + thread?: MailboxThreadInput | null + message: MailboxMessageInput + attachments?: Array + }): Promise<{ ok: true }> { + const message = input.message + assertMailboxNonEmptyString(message.id, 'message.id') + const attachments = Array.isArray(input.attachments) + ? input.attachments + : [] + this.ctx.storage.transactionSync(() => { + if (input.thread) { + this.store.upsertThreadRow(input.thread) + } + this.store.upsertMessageRow(message) + this.store.replaceAttachmentsForMessage(message.id, attachments) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { ok: true } + } + + /** + * Upsert a delivery event (idempotent on `provider_event_id`) and optionally + * apply a monotonic latest `delivery_status` update on the message. + */ + async upsertDeliveryEvent(input: { + event: MailboxDeliveryEventInput + latestDeliveryStatus?: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + } | null + }): Promise<{ inserted: boolean; updatedLatestStatus: boolean }> { + let inserted = false + let updatedLatestStatus = false + this.ctx.storage.transactionSync(() => { + inserted = this.store.writeDeliveryEventRow(input.event) + if (input.latestDeliveryStatus) { + const eventId = assertMailboxNonEmptyString(input.event.id, 'event.id') + const messageId = assertMailboxNonEmptyString( + input.latestDeliveryStatus.messageId, + 'latestDeliveryStatus.messageId', + ) + if (this.store.deliveryEventOwnsMessage(eventId, messageId)) { + updatedLatestStatus = this.store.updateLatestDeliveryStatus( + input.latestDeliveryStatus, + ) + } + } + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { inserted, updatedLatestStatus } + } + + async getThread(input: { + threadId: string + }): Promise { + return this.store.getThread(input.threadId) + } + + async getMessage(input: { + messageId: string + }): Promise { + return this.store.getMessage(input.messageId) + } + + async getMessageByMessageIdHeader(input: { + messageIdHeader: string + }): Promise { + return this.store.getMessageByMessageIdHeader(input.messageIdHeader) + } + + async getOutboundMessageByProviderMessageId(input: { + providerMessageId: string + }): Promise { + return this.store.getOutboundMessageByProviderMessageId( + input.providerMessageId, + ) + } + + async listMessages(input: MailboxListMessagesInput): Promise<{ + messages: Array + nextCursor: string | null + }> { + return this.store.listMessages(input) + } + + async searchMessages(input: MailboxSearchMessagesInput): Promise<{ + messages: Array + }> { + return this.store.searchMessages(input) + } + + async listAttachmentsForMessage(input: { + messageId: string + }): Promise> { + return this.store.listAttachmentsForMessage(input.messageId) + } + + async listDeliveryEvents(input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }): Promise> { + return this.store.listDeliveryEvents(input) + } + + async countMailbox(): Promise { + return this.store.countMailbox() + } + + async exportMailbox(input: { + pageSize?: number + startAfter?: string | null + }): Promise { + return this.store.exportMailbox(input) + } + + async listBlobReferences(input: { + pageSize?: number + startAfter?: string | null + }): Promise { + return this.store.listBlobReferences(input) + } + + /** + * Clear SQLite state only. During expand, D1 deletion remains authoritative + * for R2 objects. + */ + async purge(): Promise<{ ok: true }> { + await this.ctx.blockConcurrencyWhile(async () => { + await this.ctx.storage.deleteAlarm().catch(() => { + // Best effort: deleteAll below still clears persisted alarm state. + }) + await this.ctx.storage.deleteAll() + this.retentionAlarmArmed = false + this.retentionIdleConfirmed = true + this.store.initializeSchema() + }) + return { ok: true } + } +} + +export const Mailbox = Sentry.instrumentDurableObjectWithSentry( + (env: Env) => buildSentryOptions(env), + MailboxBase, +) + +export type { MailboxRpc } from './mailbox-types.ts' +export { + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionRetryDelayMs, + type MailboxAttachmentInput, + type MailboxAttachmentRecord, + type MailboxBlobReference, + type MailboxBlobReferencePage, + type MailboxCountResult, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, + type MailboxExportResult, + type MailboxExportRow, + type MailboxMessageInput, + type MailboxMessageRecord, + type MailboxThreadInput, + type MailboxThreadRecord, +} from './mailbox-types.ts' diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts new file mode 100644 index 0000000000..fe6b423dd7 --- /dev/null +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -0,0 +1,660 @@ +import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { + Mailbox, + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionRetryDelayMs, + type MailboxAttachmentInput, + type MailboxMessageInput, + type MailboxThreadInput, +} from './mailbox-do.ts' + +function mailboxEnv(): MailboxEnv & { MAILBOX: DurableObjectNamespace } { + const mailbox = (env as MailboxEnv).MAILBOX + if (!mailbox) { + throw new Error( + 'MAILBOX Durable Object binding is required for mailbox-do workers tests.', + ) + } + return { MAILBOX: mailbox } +} + +function stubFor(userId: string) { + const { MAILBOX } = mailboxEnv() + return MAILBOX.get(MAILBOX.idFromName(mailboxDurableObjectName(userId))) +} + +function rpcFor(userId: string) { + return mailboxRpc({ env: mailboxEnv(), userId }) +} + +function uniqueUserId(label: string) { + return `mailbox-${label}-${crypto.randomUUID()}` +} + +async function assertMailboxThrows( + pattern: RegExp, + run: () => Promise, +) { + try { + await run() + throw new Error(`Expected mailbox RPC to throw matching ${pattern}`) + } catch (error) { + expect(String(error)).toMatch(pattern) + } +} + +function baseThread( + overrides?: Partial, +): MailboxThreadInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' + return { + id, + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: ``, + lastMessageAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + } +} + +function baseMessage( + overrides?: Partial, +): MailboxMessageInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.createdAt ?? '2026-07-01T12:00:00.000Z' + return { + id, + direction: 'inbound', + inboxId: 'inbox-1', + threadId: null, + fromAddress: 'sender@example.com', + envelopeFrom: 'envelope@example.com', + toAddresses: ['owner@example.com'], + subject: 'Hello mailbox', + messageIdHeader: ``, + rawMimeKey: `email-raw:v1:owner/${id}`, + rawSize: 128, + processingStatus: 'stored', + classification: 'accepted', + receivedAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + } +} + +function baseAttachment( + messageId: string, + overrides?: Partial, +): MailboxAttachmentInput { + const id = overrides?.id ?? crypto.randomUUID() + return { + id, + messageId, + filename: 'note.txt', + contentType: 'text/plain', + size: 12, + storageKind: 'external', + storageKey: `email-attachment:v1:owner/${messageId}/${id}`, + createdAt: '2026-07-01T12:00:00.000Z', + ...overrides, + } +} + +test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', async () => { + silenceIncidentalRuntimeWarnings() + const ownerA = uniqueUserId('a') + const ownerB = uniqueUserId('b') + const mailboxA = rpcFor(ownerA) + const mailboxB = rpcFor(ownerB) + + const stubA = stubFor(ownerA) + await runInDurableObject(stubA, async (instance: Mailbox, state) => { + expect(instance).toBeInstanceOf(Mailbox) + const tables = state.storage.sql + .exec<{ name: string }>( + `SELECT name FROM sqlite_master + WHERE type = 'table' + AND name IN ( + 'email_threads', 'email_messages', + 'email_attachments', 'email_delivery_events' + ) + ORDER BY name ASC`, + ) + .toArray() + .map((row) => row.name) + expect(tables).toEqual([ + 'email_attachments', + 'email_delivery_events', + 'email_messages', + 'email_threads', + ]) + expect(await state.storage.getAlarm()).toBeNull() + }) + + const thread = baseThread({ id: 'thread-1' }) + const message = baseMessage({ + id: 'msg-1', + threadId: thread.id, + subject: 'Project update', + fromAddress: 'alice@example.com', + }) + const attachment = baseAttachment(message.id, { id: 'att-1' }) + + await mailboxA.mirrorMessage({ + thread, + message, + attachments: [attachment], + }) + await mailboxA.mirrorMessage({ + thread, + message: { ...message, subject: 'Project update (edited)' }, + attachments: [attachment], + }) + + const stored = await mailboxA.getMessage({ messageId: message.id }) + expect(stored).toMatchObject({ + id: message.id, + threadId: thread.id, + subject: 'Project update (edited)', + fromAddress: 'alice@example.com', + rawMimeKey: message.rawMimeKey, + }) + expect(await mailboxA.getThread({ threadId: thread.id })).toMatchObject({ + id: thread.id, + subjectNormalized: 'hello', + }) + expect( + await mailboxA.listAttachmentsForMessage({ messageId: message.id }), + ).toEqual([ + expect.objectContaining({ + id: attachment.id, + storageKey: attachment.storageKey, + storageKind: 'external', + }), + ]) + expect( + await mailboxA.getMessageByMessageIdHeader({ + messageIdHeader: message.messageIdHeader!, + }), + ).toMatchObject({ id: message.id }) + + const listed = await mailboxA.listMessages({ limit: 10 }) + expect(listed.messages.map((row) => row.id)).toEqual([message.id]) + const searched = await mailboxA.searchMessages({ query: 'project' }) + expect(searched.messages.map((row) => row.id)).toEqual([message.id]) + expect( + (await mailboxA.searchMessages({ query: 'nope' })).messages, + ).toHaveLength(0) + + await mailboxB.mirrorMessage({ + thread: baseThread({ id: 'thread-b' }), + message: baseMessage({ + id: 'msg-b', + threadId: 'thread-b', + subject: 'Other owner', + }), + }) + expect(await mailboxB.getMessage({ messageId: message.id })).toBeNull() + expect(await mailboxA.getMessage({ messageId: 'msg-b' })).toBeNull() + expect(await mailboxA.countMailbox()).toMatchObject({ + threads: 1, + messages: 1, + attachments: 1, + deliveryEvents: 0, + }) + expect(await mailboxB.countMailbox()).toMatchObject({ + threads: 1, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }) + + const outboundId = 'msg-out-1' + await mailboxA.mirrorMessage({ + message: baseMessage({ + id: outboundId, + direction: 'outbound', + subject: 'Sent mail', + providerMessageId: 'provider-1', + processingStatus: 'sent', + rawMimeKey: null, + }), + }) + expect( + await mailboxA.getOutboundMessageByProviderMessageId({ + providerMessageId: 'provider-1', + }), + ).toMatchObject({ id: outboundId }) +}) + +test('Mailbox delivery status, promoted inbound fields, export paging, and cursor rejection', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('export') + const mailbox = rpcFor(userId) + + const thread = baseThread({ id: 'export-thread' }) + const message = baseMessage({ + id: 'export-msg', + threadId: thread.id, + direction: 'outbound', + providerMessageId: 'prov-export', + processingStatus: 'sent', + rawMimeKey: 'email-raw:v1:owner/export-msg', + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + const attachment = baseAttachment(message.id, { + id: 'export-att', + storageKey: 'email-attachment:v1:owner/export-msg/export-att', + }) + await mailbox.mirrorMessage({ + thread, + message, + attachments: [attachment], + }) + + // Stale dual-write replay must not regress a newer delivery status. + await mailbox.mirrorMessage({ + message: { + ...message, + deliveryStatus: 'deferred', + deliveryStatusAt: '2026-07-02T09:00:00.000Z', + }, + attachments: [attachment], + }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + + // Equal timestamps may update (matches D1 <= semantics). + await mailbox.mirrorMessage({ + message: { + ...message, + deliveryStatus: 'complained', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }, + attachments: [attachment], + }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'complained', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + + const first = await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-1', + messageId: message.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'prov-export', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: false, + }, + latestDeliveryStatus: { + messageId: message.id, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:30:00.000Z', + }, + }) + expect(first).toEqual({ inserted: true, updatedLatestStatus: true }) + + const inbound = await mailbox.upsertDeliveryEvent({ + event: { + id: 'inbound-delivery-1', + messageId: 'inbound-msg-1', + inboxId: 'inbox-1', + eventType: 'received', + provider: 'cloudflare-email-routing', + createdAt: '2026-07-02T11:00:00.000Z', + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-abc', + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + expectedAttachmentCount: 2, + finalizationToken: 'lease-token-1', + usageStartedAt: '2026-07-02T10:59:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + subscriptionEffectState: 'pending', + subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', + detailJson: JSON.stringify({ recipient: 'owner@example.com' }), + }, + }) + expect(inbound.inserted).toBe(true) + const inboundRow = ( + await mailbox.listDeliveryEvents({ + messageId: 'inbound-msg-1', + limit: 1, + }) + )[0] + expect(inboundRow).toMatchObject({ + id: 'inbound-delivery-1', + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-abc', + expectedAttachmentCount: 2, + finalizationToken: 'lease-token-1', + usageStartedAt: '2026-07-02T10:59:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + subscriptionEffectState: 'pending', + subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', + }) + expect(JSON.parse(inboundRow!.detailJson)).toMatchObject({ + recipient: 'owner@example.com', + }) + + // Default needsEffectReconcile is false unless explicitly true. + await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-default-reconcile', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + createdAt: '2026-07-02T11:30:00.000Z', + }, + }) + const defaultReconcile = ( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 20 }) + ).find((event) => event.id === 'evt-default-reconcile') + expect(defaultReconcile?.needsEffectReconcile).toBe(false) + + const duplicate = await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-1-dup', + messageId: message.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'prov-export', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T11:00:00.000Z', + }, + latestDeliveryStatus: { + messageId: message.id, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T11:00:00.000Z', + }, + }) + expect(duplicate.inserted).toBe(false) + + const stale = await mailbox.upsertDeliveryEvent({ + event: { + id: 'evt-2', + messageId: message.id, + eventType: 'deferred', + provider: 'cloudflare-email', + providerMessageId: 'prov-export', + providerEventId: 'provider-event-2', + createdAt: '2026-07-02T09:00:00.000Z', + }, + latestDeliveryStatus: { + messageId: message.id, + deliveryStatus: 'deferred', + deliveryStatusAt: '2026-07-02T09:00:00.000Z', + }, + }) + expect(stale).toEqual({ inserted: true, updatedLatestStatus: false }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:30:00.000Z', + }) + + // Assert validation failures inside the DO isolate so remote RPC error + // plumbing does not surface uncaught worker exceptions. + const stub = stubFor(userId) + await runInDurableObject(stub, async (instance: Mailbox) => { + await assertMailboxThrows(/export cursor is invalid/, () => + instance.exportMailbox({ pageSize: 1, startAfter: 'not-a-cursor' }), + ) + await assertMailboxThrows(/list cursor is invalid/, () => + instance.listMessages({ limit: 1, cursor: '%%%' }), + ) + await assertMailboxThrows(/blob-reference cursor is invalid/, () => + instance.listBlobReferences({ pageSize: 1, startAfter: 'bad' }), + ) + await assertMailboxThrows(/canonical ISO-8601/, () => + instance.mirrorMessage({ + message: baseMessage({ + id: 'bad-iso', + createdAt: '2026-07-01T12:00:00Z', + }), + }), + ) + }) + + const kinds: Array = [] + let startAfter: string | null = null + for (let page = 0; page < 30; page += 1) { + const exported = await mailbox.exportMailbox({ + pageSize: 1, + startAfter, + }) + for (const row of exported.rows) { + kinds.push(row.kind) + switch (row.kind) { + case 'thread': + expect(row.row.id).toBe(thread.id) + break + case 'message': + expect(row.row.id).toBe(message.id) + break + case 'attachment': + expect(row.row.id).toBe(attachment.id) + break + case 'delivery_event': + expect(typeof row.row.id).toBe('string') + break + default: { + const exhaustive: never = row + throw new Error(`Unhandled export kind: ${String(exhaustive)}`) + } + } + } + if (!exported.truncated) break + expect(exported.nextStartAfter).not.toBe(startAfter) + startAfter = exported.nextStartAfter + } + expect(kinds.filter((kind) => kind === 'thread')).toHaveLength(1) + expect(kinds.filter((kind) => kind === 'message')).toHaveLength(1) + expect(kinds.filter((kind) => kind === 'attachment')).toHaveLength(1) + expect( + kinds.filter((kind) => kind === 'delivery_event').length, + ).toBeGreaterThanOrEqual(3) + + const blobKinds: Array = [] + let blobCursor: string | null = null + for (let page = 0; page < 10; page += 1) { + const pageResult = await mailbox.listBlobReferences({ + pageSize: 1, + startAfter: blobCursor, + }) + for (const reference of pageResult.references) { + blobKinds.push(reference.kind) + if (reference.kind === 'raw_mime') { + expect(reference.key).toBe(message.rawMimeKey) + expect(reference.messageId).toBe(message.id) + } else { + expect(reference.key).toBe(attachment.storageKey) + expect(reference.attachmentId).toBe(attachment.id) + } + } + if (!pageResult.truncated) break + expect(pageResult.nextStartAfter).not.toBe(blobCursor) + blobCursor = pageResult.nextStartAfter + } + expect(blobKinds).toEqual(['raw_mime', 'attachment']) + + await mailbox.purge() + expect(await mailbox.countMailbox()).toEqual({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) +}) + +test('Mailbox retention deletes R2 blobs before metadata and backs off on failure', async () => { + silenceIncidentalRuntimeWarnings() + consoleWarn.mockImplementation((...args: Array) => { + const message = String(args[0] ?? '') + if (message.includes('mailbox-retention-blob-delete-failed')) return + }) + + const userId = uniqueUserId('retention') + const mailbox = rpcFor(userId) + const stub = stubFor(userId) + + const oldMessageAt = new Date( + Date.now() - (mailboxMessageRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const oldEventAt = new Date( + Date.now() - (mailboxDeliveryEventRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const freshAt = new Date().toISOString() + + const keepMessage = baseMessage({ + id: 'keep-msg', + subject: 'fresh', + createdAt: freshAt, + updatedAt: freshAt, + rawMimeKey: 'email-raw:v1:owner/keep-msg', + }) + const dropMessage = baseMessage({ + id: 'drop-msg', + threadId: 'drop-thread', + subject: 'old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + rawMimeKey: 'email-raw:v1:owner/drop-msg', + }) + const failMessage = baseMessage({ + id: 'fail-msg', + subject: 'fail-old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + rawMimeKey: 'email-raw:v1:owner/fail-msg', + }) + const dropAttachment = baseAttachment(dropMessage.id, { + id: 'drop-att', + storageKey: 'email-attachment:v1:owner/drop-msg/drop-att', + createdAt: oldMessageAt, + }) + + await mailbox.mirrorMessage({ + thread: baseThread({ + id: 'drop-thread', + lastMessageAt: oldMessageAt, + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }), + message: dropMessage, + attachments: [dropAttachment], + }) + await mailbox.mirrorMessage({ message: keepMessage }) + await mailbox.mirrorMessage({ message: failMessage }) + await mailbox.upsertDeliveryEvent({ + event: { + id: 'old-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: oldEventAt, + needsEffectReconcile: false, + }, + }) + await mailbox.upsertDeliveryEvent({ + event: { + id: 'fresh-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: freshAt, + needsEffectReconcile: false, + }, + }) + + await env.EMAIL_BLOBS.put(dropMessage.rawMimeKey!, 'drop-raw') + await env.EMAIL_BLOBS.put(dropAttachment.storageKey!, 'drop-att') + await env.EMAIL_BLOBS.put(failMessage.rawMimeKey!, 'fail-raw') + await env.EMAIL_BLOBS.put(keepMessage.rawMimeKey!, 'keep-raw') + + const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) + env.EMAIL_BLOBS.delete = (async (keys: string | Array) => { + const list = Array.isArray(keys) ? keys : [keys] + if (list.includes(failMessage.rawMimeKey!)) { + throw new Error('simulated R2 delete failure') + } + return await originalDelete(keys) + }) as typeof env.EMAIL_BLOBS.delete + + try { + const alarmBefore = Date.now() + const alarmAfterFailure = await runInDurableObject( + stub, + async (instance: Mailbox, state) => { + expect(instance).toBeInstanceOf(Mailbox) + await state.storage.deleteAlarm() + await instance.alarm() + return await state.storage.getAlarm() + }, + ) + // Overdue retained work (failed R2 delete) must retry with hourly backoff, + // not every second. + expect(alarmAfterFailure).toBeTypeOf('number') + expect(alarmAfterFailure).toBeGreaterThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs - 5_000, + ) + expect(alarmAfterFailure).toBeLessThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs + 5_000, + ) + + expect(await env.EMAIL_BLOBS.get(dropMessage.rawMimeKey!)).toBeNull() + expect(await env.EMAIL_BLOBS.get(dropAttachment.storageKey!)).toBeNull() + expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).not.toBeNull() + expect(await env.EMAIL_BLOBS.get(keepMessage.rawMimeKey!)).not.toBeNull() + + expect(await mailbox.getMessage({ messageId: dropMessage.id })).toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: dropMessage.id }), + ).toHaveLength(0) + expect(await mailbox.getThread({ threadId: 'drop-thread' })).toBeNull() + expect( + await mailbox.getMessage({ messageId: failMessage.id }), + ).toMatchObject({ + id: failMessage.id, + rawMimeKey: failMessage.rawMimeKey, + }) + expect( + await mailbox.getMessage({ messageId: keepMessage.id }), + ).toMatchObject({ + id: keepMessage.id, + }) + const events = await mailbox.listDeliveryEvents({ limit: 10 }) + expect(events.map((event) => event.id)).toEqual(['fresh-event']) + } finally { + env.EMAIL_BLOBS.delete = originalDelete + } + + await runInDurableObject(stub, async (instance: Mailbox) => { + await instance.alarm() + }) + expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).toBeNull() + expect(await mailbox.getMessage({ messageId: failMessage.id })).toBeNull() +}) diff --git a/packages/worker/src/email/mailbox-export.ts b/packages/worker/src/email/mailbox-export.ts new file mode 100644 index 0000000000..e2202c6999 --- /dev/null +++ b/packages/worker/src/email/mailbox-export.ts @@ -0,0 +1,293 @@ +import { + mailboxBlobRefAttachmentCursorPrefix, + mailboxBlobRefRawMimeCursorPrefix, + mailboxExportCursorPrefixForPhase, + normalizeMailboxPageSize, + parseMailboxBlobRefCursor, + parseMailboxExportCursor, + type MailboxBlobReference, + type MailboxBlobReferencePage, + type MailboxExportPhase, + type MailboxExportResult, + type MailboxExportRow, +} from './mailbox-types.ts' +import { + mapMailboxAttachmentRow, + mapMailboxDeliveryEventRow, + mapMailboxMessageRow, + mapMailboxThreadRow, +} from './mailbox-mappers.ts' + +export function exportMailboxFromStore( + sql: SqlStorage, + input: { + pageSize?: number + startAfter?: string | null + }, +): MailboxExportResult { + const pageSize = normalizeMailboxPageSize(input.pageSize) + const startAfterRaw = + typeof input.startAfter === 'string' && input.startAfter.length > 0 + ? input.startAfter + : null + const phaseOrder: Array = [ + 'thread', + 'message', + 'attachment', + 'delivery_event', + ] + const result: MailboxExportResult = { + rows: [], + nextStartAfter: null, + truncated: false, + } + let { phase, startAfterId } = parseMailboxExportCursor(startAfterRaw) + let remaining = pageSize + let phaseIndex = phaseOrder.indexOf(phase) + const maxIterations = phaseOrder.length + 1 + let iterations = 0 + + while (phaseIndex < phaseOrder.length) { + iterations += 1 + if (iterations > maxIterations) { + throw new Error('exportMailbox exceeded phase iteration budget') + } + if (remaining <= 0) { + const nextPhase = phaseOrder[phaseIndex]! + const handoff = mailboxExportCursorPrefixForPhase(nextPhase) + if (handoff === startAfterRaw) { + phaseIndex += 1 + startAfterId = '' + continue + } + result.nextStartAfter = handoff + result.truncated = true + return result + } + + const currentPhase = phaseOrder[phaseIndex]! + const page = exportPhasePage(sql, { + phase: currentPhase, + startAfterId, + limit: remaining, + }) + result.rows.push(...page.rows) + remaining -= page.taken + + if (page.truncated) { + if (page.taken <= 0) { + phaseIndex += 1 + startAfterId = '' + continue + } + const nextStartAfter = `${mailboxExportCursorPrefixForPhase(currentPhase)}${page.nextId}` + if (nextStartAfter === startAfterRaw) { + phaseIndex += 1 + startAfterId = '' + continue + } + result.nextStartAfter = nextStartAfter + result.truncated = true + return result + } + + phaseIndex += 1 + startAfterId = '' + if (remaining <= 0 && phaseIndex < phaseOrder.length) { + const nextPhase = phaseOrder[phaseIndex]! + const handoff = mailboxExportCursorPrefixForPhase(nextPhase) + if (handoff === startAfterRaw) continue + result.nextStartAfter = handoff + result.truncated = true + return result + } + } + return result +} + +function exportPhasePage( + sql: SqlStorage, + input: { + phase: MailboxExportPhase + startAfterId: string + limit: number + }, +): { + rows: Array + taken: number + truncated: boolean + nextId: string +} { + if (input.limit <= 0) { + return { + rows: [], + taken: 0, + truncated: false, + nextId: input.startAfterId, + } + } + const limit = Math.trunc(input.limit) + const tableForPhase = (phase: MailboxExportPhase) => { + switch (phase) { + case 'thread': + return 'email_threads' + case 'message': + return 'email_messages' + case 'attachment': + return 'email_attachments' + case 'delivery_event': + return 'email_delivery_events' + default: { + const exhaustive: never = phase + throw new Error(`Unhandled export phase: ${String(exhaustive)}`) + } + } + } + const rows = sql + .exec>( + `SELECT * FROM ${tableForPhase(input.phase)} + WHERE id > ? + ORDER BY id ASC + LIMIT ?`, + input.startAfterId, + limit + 1, + ) + .toArray() + const truncated = rows.length > limit + const pageRows = truncated ? rows.slice(0, limit) : rows + const last = pageRows[pageRows.length - 1] + const mapped: Array = pageRows.map((row) => { + switch (input.phase) { + case 'thread': + return { kind: 'thread', row: mapMailboxThreadRow(row) } + case 'message': + return { kind: 'message', row: mapMailboxMessageRow(row) } + case 'attachment': + return { kind: 'attachment', row: mapMailboxAttachmentRow(row) } + case 'delivery_event': + return { + kind: 'delivery_event', + row: mapMailboxDeliveryEventRow(row), + } + default: { + const exhaustive: never = input.phase + throw new Error(`Unhandled export phase: ${String(exhaustive)}`) + } + } + }) + return { + rows: mapped, + taken: pageRows.length, + truncated, + nextId: last ? String(last['id']) : input.startAfterId, + } +} + +export function listMailboxBlobReferences( + sql: SqlStorage, + input: { + pageSize?: number + startAfter?: string | null + }, +): MailboxBlobReferencePage { + const pageSize = normalizeMailboxPageSize(input.pageSize) + const startAfterRaw = + typeof input.startAfter === 'string' && input.startAfter.length > 0 + ? input.startAfter + : null + const { phase, startAfterId } = parseMailboxBlobRefCursor(startAfterRaw) + const references: Array = [] + let remaining = pageSize + + if (phase === 'raw_mime') { + const rows = sql + .exec<{ id: string; raw_mime_key: string }>( + `SELECT id, raw_mime_key FROM email_messages + WHERE raw_mime_key IS NOT NULL AND id > ? + ORDER BY id ASC + LIMIT ?`, + startAfterId, + remaining + 1, + ) + .toArray() + const truncated = rows.length > remaining + const pageRows = truncated ? rows.slice(0, remaining) : rows + for (const row of pageRows) { + references.push({ + kind: 'raw_mime', + key: row.raw_mime_key, + messageId: row.id, + attachmentId: null, + }) + } + if (truncated) { + const last = pageRows[pageRows.length - 1]! + return { + references, + nextStartAfter: `${mailboxBlobRefRawMimeCursorPrefix}${last.id}`, + truncated: true, + } + } + remaining -= pageRows.length + if (remaining <= 0) { + return { + references, + nextStartAfter: mailboxBlobRefAttachmentCursorPrefix, + truncated: true, + } + } + const attachmentPage = listAttachmentBlobReferences(sql, { + startAfterId: '', + limit: remaining, + }) + references.push(...attachmentPage.references) + return { + references, + nextStartAfter: attachmentPage.nextStartAfter, + truncated: attachmentPage.truncated, + } + } + + return listAttachmentBlobReferences(sql, { + startAfterId, + limit: remaining, + }) +} + +function listAttachmentBlobReferences( + sql: SqlStorage, + input: { + startAfterId: string + limit: number + }, +): MailboxBlobReferencePage { + if (input.limit <= 0) { + return { references: [], nextStartAfter: null, truncated: false } + } + const rows = sql + .exec<{ id: string; message_id: string; storage_key: string }>( + `SELECT id, message_id, storage_key FROM email_attachments + WHERE storage_key IS NOT NULL AND id > ? + ORDER BY id ASC + LIMIT ?`, + input.startAfterId, + input.limit + 1, + ) + .toArray() + const truncated = rows.length > input.limit + const pageRows = truncated ? rows.slice(0, input.limit) : rows + const last = pageRows[pageRows.length - 1] + return { + references: pageRows.map((row) => ({ + kind: 'attachment' as const, + key: row.storage_key, + messageId: row.message_id, + attachmentId: row.id, + })), + nextStartAfter: + truncated && last + ? `${mailboxBlobRefAttachmentCursorPrefix}${last.id}` + : null, + truncated, + } +} diff --git a/packages/worker/src/email/mailbox-mappers.ts b/packages/worker/src/email/mailbox-mappers.ts new file mode 100644 index 0000000000..9934469b5c --- /dev/null +++ b/packages/worker/src/email/mailbox-mappers.ts @@ -0,0 +1,319 @@ +import { + maxRestorableTextColumnBytes, + truncateToUtf8Bytes, + utf8ByteLength, +} from '@kody-internal/shared/backup-restore-safety.ts' +import { parseJsonArray } from '@kody-internal/shared/json-parsing.ts' +import { + emailClassificationValues, + emailDeliveryEventTypeValues, + emailDeliveryStatusValues, + emailDirectionValues, + emailProcessingStatusValues, + type EmailClassification, + type EmailDeliveryEventType, + type EmailDeliveryStatus, + type EmailDirection, + type EmailProcessingStatus, +} from './types.ts' +import { + mailboxInboundDeliveryStateValues, + mailboxSubscriptionEffectStateValues, + type MailboxAttachmentRecord, + type MailboxDeliveryEventRecord, + type MailboxInboundDeliveryState, + type MailboxMessageRecord, + type MailboxSubscriptionEffectState, + type MailboxThreadRecord, +} from './mailbox-types.ts' + +const emailBodyTruncationNotice = + '\n[truncated for backup-safe storage; the full message is retained in the raw MIME object]' + +export function boundedEmailBody( + body: string | null | undefined, +): string | null { + if (body == null) return null + if (utf8ByteLength(body) <= maxRestorableTextColumnBytes) return body + return ( + truncateToUtf8Bytes( + body, + maxRestorableTextColumnBytes - utf8ByteLength(emailBodyTruncationNotice), + ) + emailBodyTruncationNotice + ) +} + +function parseOptionalJsonRecord(value: string | null) { + if (!value) return null + try { + const parsed = JSON.parse(value) as unknown + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return null + } + return parsed as Record + } catch { + return null + } +} + +export function escapeLikePattern(value: string) { + return value.replaceAll(/[\\%_]/g, (character) => `\\${character}`) +} + +function mapInboundState(value: unknown): MailboxInboundDeliveryState | null { + if (value == null) return null + const state = String(value) + return (mailboxInboundDeliveryStateValues as ReadonlyArray).includes( + state, + ) + ? (state as MailboxInboundDeliveryState) + : null +} + +function mapSubscriptionEffectState( + value: unknown, +): MailboxSubscriptionEffectState | null { + if (value == null) return null + const state = String(value) + return ( + mailboxSubscriptionEffectStateValues as ReadonlyArray + ).includes(state) + ? (state as MailboxSubscriptionEffectState) + : null +} + +export function mapMailboxThreadRow( + row: Record, +): MailboxThreadRecord { + return { + id: String(row['id']), + inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), + subjectNormalized: + row['subject_normalized'] == null + ? null + : String(row['subject_normalized']), + rootMessageIdHeader: + row['root_message_id_header'] == null + ? null + : String(row['root_message_id_header']), + lastMessageAt: String(row['last_message_at']), + createdAt: String(row['created_at']), + updatedAt: String(row['updated_at']), + } +} + +export function mapMailboxMessageRow( + row: Record, +): MailboxMessageRecord { + const direction = String(row['direction'] ?? '') + const processingStatus = String(row['processing_status'] ?? '') + const classificationRaw = String(row['classification'] ?? 'accepted') + const deliveryStatusRaw = + row['delivery_status'] == null ? null : String(row['delivery_status']) + return { + id: String(row['id']), + direction: (emailDirectionValues as ReadonlyArray).includes( + direction, + ) + ? (direction as EmailDirection) + : 'inbound', + inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), + threadId: row['thread_id'] == null ? null : String(row['thread_id']), + senderIdentityId: + row['sender_identity_id'] == null + ? null + : String(row['sender_identity_id']), + fromAddress: + row['from_address'] == null ? null : String(row['from_address']), + envelopeFrom: + row['envelope_from'] == null ? null : String(row['envelope_from']), + toAddresses: parseJsonArray(String(row['to_addresses_json'] ?? '[]')), + ccAddresses: parseJsonArray(String(row['cc_addresses_json'] ?? '[]')), + bccAddresses: parseJsonArray(String(row['bcc_addresses_json'] ?? '[]')), + replyToAddresses: parseJsonArray( + String(row['reply_to_addresses_json'] ?? '[]'), + ), + subject: row['subject'] == null ? null : String(row['subject']), + messageIdHeader: + row['message_id_header'] == null + ? null + : String(row['message_id_header']), + inReplyToHeader: + row['in_reply_to_header'] == null + ? null + : String(row['in_reply_to_header']), + references: parseJsonArray(String(row['references_json'] ?? '[]')), + headers: parseOptionalJsonRecord( + row['headers_json'] == null ? null : String(row['headers_json']), + ), + authResults: + row['auth_results'] == null ? null : String(row['auth_results']), + textBody: row['text_body'] == null ? null : String(row['text_body']), + htmlBody: row['html_body'] == null ? null : String(row['html_body']), + rawMimeKey: + row['raw_mime_key'] == null ? null : String(row['raw_mime_key']), + rawSize: row['raw_size'] == null ? null : Number(row['raw_size']) || 0, + processingStatus: ( + emailProcessingStatusValues as ReadonlyArray + ).includes(processingStatus) + ? (processingStatus as EmailProcessingStatus) + : 'stored', + classification: ( + emailClassificationValues as ReadonlyArray + ).includes(classificationRaw) + ? (classificationRaw as EmailClassification) + : 'accepted', + classificationReason: + row['classification_reason'] == null + ? null + : String(row['classification_reason']), + providerMessageId: + row['provider_message_id'] == null + ? null + : String(row['provider_message_id']), + deliveryStatus: + deliveryStatusRaw != null && + (emailDeliveryStatusValues as ReadonlyArray).includes( + deliveryStatusRaw, + ) + ? (deliveryStatusRaw as EmailDeliveryStatus) + : null, + deliveryStatusAt: + row['delivery_status_at'] == null + ? null + : String(row['delivery_status_at']), + error: row['error'] == null ? null : String(row['error']), + receivedAt: row['received_at'] == null ? null : String(row['received_at']), + sentAt: row['sent_at'] == null ? null : String(row['sent_at']), + createdAt: String(row['created_at']), + updatedAt: String(row['updated_at']), + } +} + +export function mapMailboxAttachmentRow( + row: Record, +): MailboxAttachmentRecord { + return { + id: String(row['id']), + messageId: String(row['message_id']), + filename: row['filename'] == null ? null : String(row['filename']), + contentType: + row['content_type'] == null ? null : String(row['content_type']), + contentId: row['content_id'] == null ? null : String(row['content_id']), + disposition: row['disposition'] == null ? null : String(row['disposition']), + size: Number(row['size'] ?? 0) || 0, + storageKind: String(row['storage_kind']), + storageKey: row['storage_key'] == null ? null : String(row['storage_key']), + createdAt: String(row['created_at']), + } +} + +export function mapMailboxDeliveryEventRow( + row: Record, +): MailboxDeliveryEventRecord { + const eventType = String(row['event_type'] ?? '') + return { + id: String(row['id']), + messageId: row['message_id'] == null ? null : String(row['message_id']), + inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), + eventType: (emailDeliveryEventTypeValues as ReadonlyArray).includes( + eventType, + ) + ? (eventType as EmailDeliveryEventType) + : 'received', + provider: row['provider'] == null ? null : String(row['provider']), + providerMessageId: + row['provider_message_id'] == null + ? null + : String(row['provider_message_id']), + providerEventId: + row['provider_event_id'] == null + ? null + : String(row['provider_event_id']), + detailJson: String(row['detail_json'] ?? '{}'), + needsEffectReconcile: Number(row['needs_effect_reconcile'] ?? 0) === 1, + state: mapInboundState(row['state']), + fingerprint: row['fingerprint'] == null ? null : String(row['fingerprint']), + storageLease: + row['storage_lease'] == null ? null : String(row['storage_lease']), + storageLeaseAt: + row['storage_lease_at'] == null ? null : String(row['storage_lease_at']), + cleanupLease: + row['cleanup_lease'] == null ? null : String(row['cleanup_lease']), + cleanupLeaseAt: + row['cleanup_lease_at'] == null ? null : String(row['cleanup_lease_at']), + cleanupRetryAt: + row['cleanup_retry_at'] == null ? null : String(row['cleanup_retry_at']), + expectedAttachmentCount: + row['expected_attachment_count'] == null + ? null + : Number(row['expected_attachment_count']) || 0, + finalizationToken: + row['finalization_token'] == null + ? null + : String(row['finalization_token']), + reconcileAfter: + row['reconcile_after'] == null ? null : String(row['reconcile_after']), + dedupeExpiresAt: + row['dedupe_expires_at'] == null + ? null + : String(row['dedupe_expires_at']), + usageEffectRecordedAt: + row['usage_effect_recorded_at'] == null + ? null + : String(row['usage_effect_recorded_at']), + usageEffectSuppressedAt: + row['usage_effect_suppressed_at'] == null + ? null + : String(row['usage_effect_suppressed_at']), + usageStartedAt: + row['usage_started_at'] == null ? null : String(row['usage_started_at']), + usageMonth: row['usage_month'] == null ? null : String(row['usage_month']), + usageBytes: + row['usage_bytes'] == null ? null : Number(row['usage_bytes']) || 0, + usageDurationMs: + row['usage_duration_ms'] == null + ? null + : Number(row['usage_duration_ms']) || 0, + usageEffectRetryAt: + row['usage_effect_retry_at'] == null + ? null + : String(row['usage_effect_retry_at']), + usageEffectLease: + row['usage_effect_lease'] == null + ? null + : String(row['usage_effect_lease']), + usageEffectLeaseAt: + row['usage_effect_lease_at'] == null + ? null + : String(row['usage_effect_lease_at']), + subscriptionEffectState: mapSubscriptionEffectState( + row['subscription_effect_state'], + ), + subscriptionEffectLease: + row['subscription_effect_lease'] == null + ? null + : String(row['subscription_effect_lease']), + subscriptionEffectLeaseAt: + row['subscription_effect_lease_at'] == null + ? null + : String(row['subscription_effect_lease_at']), + subscriptionEffectRetryAt: + row['subscription_effect_retry_at'] == null + ? null + : String(row['subscription_effect_retry_at']), + subscriptionEffectAttemptCount: + row['subscription_effect_attempt_count'] == null + ? null + : Number(row['subscription_effect_attempt_count']) || 0, + subscriptionEffectDeadLetterAt: + row['subscription_effect_dead_letter_at'] == null + ? null + : String(row['subscription_effect_dead_letter_at']), + subscriptionEffectLastError: + row['subscription_effect_last_error'] == null + ? null + : String(row['subscription_effect_last_error']), + createdAt: String(row['created_at']), + } +} diff --git a/packages/worker/src/email/mailbox-retention.ts b/packages/worker/src/email/mailbox-retention.ts new file mode 100644 index 0000000000..2a86713797 --- /dev/null +++ b/packages/worker/src/email/mailbox-retention.ts @@ -0,0 +1,137 @@ +import { + mailboxBlobDeleteMaxKeys, + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionBatchSize, + mailboxRetentionRetryDelayMs, +} from './mailbox-types.ts' +import { type MailboxStore } from './mailbox-store.ts' + +const messageRetentionMs = mailboxMessageRetentionDays * 24 * 60 * 60 * 1000 +const deliveryEventRetentionMs = + mailboxDeliveryEventRetentionDays * 24 * 60 * 60 * 1000 + +export type MailboxRetentionPassResult = { + /** True when at least one message blob delete failed (row retained). */ + hadBlobDeleteFailures: boolean +} + +/** + * Soonest retention due-time from stored rows, or null when empty. + * Overdue rows yield timestamps in the past — callers must apply retry backoff + * instead of scheduling at now+1s. + */ +export function nextMailboxRetentionDueAtMs( + store: MailboxStore, +): number | null { + let next: number | null = null + const consider = (at: number) => { + if (!Number.isFinite(at)) return + if (next == null || at < next) next = at + } + + const oldestMessage = store.oldestMessageCreatedAt() + if (oldestMessage) { + consider(Date.parse(oldestMessage) + messageRetentionMs) + } + const oldestEvent = store.oldestDeliveryEventCreatedAt() + if (oldestEvent) { + consider(Date.parse(oldestEvent) + deliveryEventRetentionMs) + } + return next +} + +/** + * Schedule time for the next retention alarm. + * - Future due-time → schedule at due-time + * - Overdue work → bounded hourly backoff (avoids 1s storms on persistent R2 failure) + */ +export function mailboxRetentionAlarmAtMs(input: { + dueAtMs: number | null + nowMs?: number +}): number | null { + if (input.dueAtMs == null) return null + const nowMs = input.nowMs ?? Date.now() + if (input.dueAtMs <= nowMs) { + return nowMs + mailboxRetentionRetryDelayMs + } + return input.dueAtMs +} + +async function deleteBlobKeys( + blobs: Pick, + keys: Array, +): Promise { + if (keys.length === 0) return true + try { + for ( + let index = 0; + index < keys.length; + index += mailboxBlobDeleteMaxKeys + ) { + const chunk = keys.slice(index, index + mailboxBlobDeleteMaxKeys) + await blobs.delete(chunk) + } + return true + } catch (error) { + console.warn('mailbox-retention-blob-delete-failed', { error }) + return false + } +} + +async function pruneExpiredMessages(input: { + store: MailboxStore + blobs: Pick +}): Promise { + const cutoff = new Date(Date.now() - messageRetentionMs).toISOString() + const rows = input.store.listExpiredMessagesForRetention({ + cutoff, + limit: mailboxRetentionBatchSize, + }) + if (rows.length === 0) return false + + const attachmentRows = input.store.listAttachmentStorageKeysForMessages( + rows.map((row) => row.id), + ) + const attachmentKeysByMessageId = new Map>() + for (const attachment of attachmentRows) { + const keys = attachmentKeysByMessageId.get(attachment.message_id) + if (keys) keys.push(attachment.storage_key) + else { + attachmentKeysByMessageId.set(attachment.message_id, [ + attachment.storage_key, + ]) + } + } + + let hadBlobDeleteFailures = false + for (const row of rows) { + const keys = [ + ...(row.raw_mime_key ? [row.raw_mime_key] : []), + ...(attachmentKeysByMessageId.get(row.id) ?? []), + ] + const blobsDeleted = await deleteBlobKeys(input.blobs, keys) + if (!blobsDeleted) { + hadBlobDeleteFailures = true + continue + } + input.store.deleteMessageCascade(row.id) + } + return hadBlobDeleteFailures +} + +export async function enforceMailboxRetention(input: { + store: MailboxStore + blobs: Pick +}): Promise { + const hadBlobDeleteFailures = await pruneExpiredMessages(input) + const eventCutoff = new Date( + Date.now() - deliveryEventRetentionMs, + ).toISOString() + input.store.pruneExpiredDeliveryEvents({ + cutoff: eventCutoff, + limit: mailboxRetentionBatchSize, + }) + input.store.pruneOrphanThreads(mailboxRetentionBatchSize) + return { hadBlobDeleteFailures } +} diff --git a/packages/worker/src/email/mailbox-schema.ts b/packages/worker/src/email/mailbox-schema.ts new file mode 100644 index 0000000000..ef8bcdda5d --- /dev/null +++ b/packages/worker/src/email/mailbox-schema.ts @@ -0,0 +1,328 @@ +import { + mailboxMetaSchemaVersionKey, + mailboxSchemaVersion, +} from './mailbox-types.ts' + +export function getMailboxMeta( + storage: DurableObjectStorage, + key: string, +): number | null { + const row = storage.sql + .exec<{ value: number }>( + `SELECT value FROM mailbox_meta WHERE key = ? LIMIT 1`, + key, + ) + .toArray()[0] + return row == null ? null : Number(row.value) || 0 +} + +export function setMailboxMeta( + storage: DurableObjectStorage, + key: string, + value: number, +) { + storage.sql.exec( + `INSERT INTO mailbox_meta (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + key, + value, + ) +} + +/** Warm v1 objects lack promoted inbound columns. */ +function ensureDeliveryEventPromotedColumns( + sql: SqlStorage, + installedVersion: number | null, +) { + if (installedVersion == null || installedVersion >= mailboxSchemaVersion) { + return + } + const columns: Array = [ + `state TEXT`, + `fingerprint TEXT`, + `storage_lease TEXT`, + `storage_lease_at TEXT`, + `cleanup_lease TEXT`, + `cleanup_lease_at TEXT`, + `cleanup_retry_at TEXT`, + `expected_attachment_count INTEGER`, + `finalization_token TEXT`, + `reconcile_after TEXT`, + `dedupe_expires_at TEXT`, + `usage_effect_suppressed_at TEXT`, + `usage_started_at TEXT`, + `usage_effect_retry_at TEXT`, + `usage_effect_lease TEXT`, + `usage_effect_lease_at TEXT`, + `subscription_effect_state TEXT`, + `subscription_effect_lease TEXT`, + `subscription_effect_lease_at TEXT`, + `subscription_effect_retry_at TEXT`, + `subscription_effect_attempt_count INTEGER`, + `subscription_effect_dead_letter_at TEXT`, + `subscription_effect_last_error TEXT`, + ] + for (const column of columns) { + try { + sql.exec(`ALTER TABLE email_delivery_events ADD COLUMN ${column}`) + } catch { + // Column already present on a partially migrated object. + } + } +} + +export function initializeMailboxSchema(storage: DurableObjectStorage) { + const sql = storage.sql + sql.exec(` + CREATE TABLE IF NOT EXISTS mailbox_meta ( + key TEXT PRIMARY KEY NOT NULL, + value INTEGER NOT NULL + ) + `) + const installedVersion = getMailboxMeta(storage, mailboxMetaSchemaVersionKey) + if (installedVersion === mailboxSchemaVersion) return + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_threads ( + id TEXT PRIMARY KEY NOT NULL, + inbox_id TEXT, + subject_normalized TEXT NOT NULL DEFAULT '', + root_message_id_header TEXT, + last_message_at TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + `) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_threads_last_message_at + ON email_threads(last_message_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_threads_root_message_id + ON email_threads(root_message_id_header) + WHERE root_message_id_header IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_threads_created_at + ON email_threads(created_at ASC, id ASC)`, + ) + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_messages ( + id TEXT PRIMARY KEY NOT NULL, + direction TEXT NOT NULL CHECK (direction IN ('inbound', 'outbound')), + inbox_id TEXT, + thread_id TEXT, + sender_identity_id TEXT, + from_address TEXT NOT NULL DEFAULT '', + envelope_from TEXT, + to_addresses_json TEXT NOT NULL DEFAULT '[]', + cc_addresses_json TEXT NOT NULL DEFAULT '[]', + bcc_addresses_json TEXT NOT NULL DEFAULT '[]', + reply_to_addresses_json TEXT NOT NULL DEFAULT '[]', + subject TEXT NOT NULL DEFAULT '', + message_id_header TEXT, + in_reply_to_header TEXT, + references_json TEXT NOT NULL DEFAULT '[]', + headers_json TEXT NOT NULL DEFAULT '{}', + auth_results TEXT, + text_body TEXT, + html_body TEXT, + raw_mime_key TEXT, + raw_size INTEGER NOT NULL DEFAULT 0, + processing_status TEXT NOT NULL CHECK ( + processing_status IN ('stored', 'sent', 'failed') + ), + classification TEXT NOT NULL DEFAULT 'accepted' CHECK ( + classification IN ('accepted', 'quarantined') + ), + classification_reason TEXT, + provider_message_id TEXT, + delivery_status TEXT CHECK ( + delivery_status IS NULL OR delivery_status IN ( + 'delivered', 'deferred', 'bounced', 'failed', + 'rejected', 'complained' + ) + ), + delivery_status_at TEXT, + error TEXT, + received_at TEXT, + sent_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + `) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_created_at + ON email_messages(created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_created_at_asc + ON email_messages(created_at ASC, id ASC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_inbox_created_at + ON email_messages(inbox_id, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_thread_created_at + ON email_messages(thread_id, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_message_id_header + ON email_messages(message_id_header) + WHERE message_id_header IS NOT NULL`, + ) + sql.exec( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_email_messages_provider_message_id + ON email_messages(provider_message_id) + WHERE direction = 'outbound' AND provider_message_id IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_direction_created_at + ON email_messages(direction, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_delivery_status_created_at + ON email_messages(delivery_status, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_classification_created_at + ON email_messages(classification, created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_messages_raw_mime_key + ON email_messages(id ASC) + WHERE raw_mime_key IS NOT NULL`, + ) + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_attachments ( + id TEXT PRIMARY KEY NOT NULL, + message_id TEXT NOT NULL, + filename TEXT, + content_type TEXT NOT NULL DEFAULT 'application/octet-stream', + content_id TEXT, + disposition TEXT, + size INTEGER NOT NULL DEFAULT 0, + storage_kind TEXT NOT NULL CHECK ( + storage_kind IN ('raw-mime', 'external', 'unavailable') + ), + storage_key TEXT, + created_at TEXT NOT NULL + ) + `) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_attachments_message_id + ON email_attachments(message_id)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_attachments_storage_key + ON email_attachments(id ASC) + WHERE storage_key IS NOT NULL`, + ) + + sql.exec(` + CREATE TABLE IF NOT EXISTS email_delivery_events ( + id TEXT PRIMARY KEY NOT NULL, + message_id TEXT, + inbox_id TEXT, + event_type TEXT NOT NULL CHECK ( + event_type IN ( + 'receive_started', 'received', 'rejected', 'send_requested', + 'sent', 'failed', 'delivered', 'deferred', 'bounced', 'complained' + ) + ), + provider TEXT NOT NULL DEFAULT 'kody', + provider_message_id TEXT, + provider_event_id TEXT, + detail_json TEXT NOT NULL DEFAULT '{}', + needs_effect_reconcile INTEGER NOT NULL DEFAULT 0 CHECK ( + needs_effect_reconcile IN (0, 1) + ), + state TEXT CHECK ( + state IS NULL OR state IN ( + 'pending', 'storing', 'cleaning', 'received', + 'rejected', 'orphan-cleaned' + ) + ), + fingerprint TEXT, + storage_lease TEXT, + storage_lease_at TEXT, + cleanup_lease TEXT, + cleanup_lease_at TEXT, + cleanup_retry_at TEXT, + expected_attachment_count INTEGER, + finalization_token TEXT, + reconcile_after TEXT, + dedupe_expires_at TEXT, + usage_effect_recorded_at TEXT, + usage_effect_suppressed_at TEXT, + usage_started_at TEXT, + usage_month TEXT, + usage_bytes INTEGER, + usage_duration_ms INTEGER, + usage_effect_retry_at TEXT, + usage_effect_lease TEXT, + usage_effect_lease_at TEXT, + subscription_effect_state TEXT CHECK ( + subscription_effect_state IS NULL OR subscription_effect_state IN ( + 'pending', 'processing', 'complete', 'dead-letter' + ) + ), + subscription_effect_lease TEXT, + subscription_effect_lease_at TEXT, + subscription_effect_retry_at TEXT, + subscription_effect_attempt_count INTEGER, + subscription_effect_dead_letter_at TEXT, + subscription_effect_last_error TEXT, + created_at TEXT NOT NULL + ) + `) + ensureDeliveryEventPromotedColumns(sql, installedVersion) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_message_id + ON email_delivery_events(message_id)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_created_at + ON email_delivery_events(created_at DESC, id DESC)`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_created_at_asc + ON email_delivery_events(created_at ASC, id ASC)`, + ) + sql.exec( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_email_delivery_events_provider_event_id + ON email_delivery_events(provider_event_id) + WHERE provider_event_id IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_pending_effects + ON email_delivery_events(created_at ASC, id ASC) + WHERE needs_effect_reconcile = 1 + AND fingerprint IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_state_created + ON email_delivery_events(state, created_at ASC) + WHERE state IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_fingerprint + ON email_delivery_events(fingerprint) + WHERE fingerprint IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_dedupe_expires + ON email_delivery_events(dedupe_expires_at ASC) + WHERE dedupe_expires_at IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_recorded_usage_month + ON email_delivery_events(usage_month, created_at ASC) + WHERE usage_effect_recorded_at IS NOT NULL`, + ) + + setMailboxMeta(storage, mailboxMetaSchemaVersionKey, mailboxSchemaVersion) +} diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts new file mode 100644 index 0000000000..ce5c591766 --- /dev/null +++ b/packages/worker/src/email/mailbox-store.ts @@ -0,0 +1,840 @@ +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxClassification, + assertMailboxDeliveryEventType, + assertMailboxDeliveryStatus, + assertMailboxDirection, + assertMailboxInboundDeliveryState, + assertMailboxNonEmptyString, + assertMailboxProcessingStatus, + assertMailboxStorageKind, + assertMailboxSubscriptionEffectState, + assertOptionalMailboxCanonicalIsoTimestamp, + decodeMailboxListCursor, + encodeMailboxListCursor, + mailboxNowIso, + normalizeMailboxPageSize, + type MailboxAttachmentInput, + type MailboxAttachmentRecord, + type MailboxBlobReferencePage, + type MailboxCountResult, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, + type MailboxExportResult, + type MailboxListMessagesInput, + type MailboxMessageInput, + type MailboxMessageRecord, + type MailboxSearchMessagesInput, + type MailboxThreadInput, + type MailboxThreadRecord, +} from './mailbox-types.ts' +import { + type EmailDeliveryEventType, + type EmailDeliveryStatus, +} from './types.ts' +import { + boundedEmailBody, + escapeLikePattern, + mapMailboxAttachmentRow, + mapMailboxDeliveryEventRow, + mapMailboxMessageRow, + mapMailboxThreadRow, +} from './mailbox-mappers.ts' +import { + exportMailboxFromStore, + listMailboxBlobReferences, +} from './mailbox-export.ts' +import { initializeMailboxSchema } from './mailbox-schema.ts' + +/** + * SQLite write/query helpers for one Mailbox DO. No alarm / R2 side effects. + */ +export class MailboxStore { + private readonly storage: DurableObjectStorage + + constructor(storage: DurableObjectStorage) { + this.storage = storage + } + + private get sql() { + return this.storage.sql + } + + initializeSchema() { + initializeMailboxSchema(this.storage) + } + + upsertThreadRow(thread: MailboxThreadInput) { + const id = assertMailboxNonEmptyString(thread.id, 'thread.id') + const lastMessageAt = assertMailboxCanonicalIsoTimestamp( + thread.lastMessageAt, + 'thread.lastMessageAt', + ) + const createdAt = assertMailboxCanonicalIsoTimestamp( + thread.createdAt?.trim() || mailboxNowIso(), + 'thread.createdAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + thread.updatedAt?.trim() || mailboxNowIso(), + 'thread.updatedAt', + ) + this.sql.exec( + `INSERT INTO email_threads ( + id, inbox_id, subject_normalized, root_message_id_header, + last_message_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + inbox_id = excluded.inbox_id, + subject_normalized = excluded.subject_normalized, + root_message_id_header = excluded.root_message_id_header, + last_message_at = excluded.last_message_at, + updated_at = excluded.updated_at`, + id, + thread.inboxId ?? null, + thread.subjectNormalized ?? '', + thread.rootMessageIdHeader ?? null, + lastMessageAt, + createdAt, + updatedAt, + ) + } + + /** + * Upsert message metadata. Delivery status is monotonic by + * `delivery_status_at` (`>=` / equal timestamps may update; older must not + * regress a newer status from dual-write replay). + */ + upsertMessageRow(message: MailboxMessageInput) { + const id = assertMailboxNonEmptyString(message.id, 'message.id') + const direction = assertMailboxDirection(message.direction) + const processingStatus = assertMailboxProcessingStatus( + message.processingStatus, + ) + const classification = assertMailboxClassification( + message.classification ?? 'accepted', + ) + const deliveryStatus = + message.deliveryStatus == null + ? null + : assertMailboxDeliveryStatus(message.deliveryStatus) + const deliveryStatusAt = assertOptionalMailboxCanonicalIsoTimestamp( + message.deliveryStatusAt, + 'message.deliveryStatusAt', + ) + if (deliveryStatus != null && deliveryStatusAt == null) { + throw new Error( + 'Mailbox message.deliveryStatusAt is required when deliveryStatus is set.', + ) + } + const createdAt = assertMailboxCanonicalIsoTimestamp( + message.createdAt?.trim() || mailboxNowIso(), + 'message.createdAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + message.updatedAt?.trim() || mailboxNowIso(), + 'message.updatedAt', + ) + const receivedAt = assertOptionalMailboxCanonicalIsoTimestamp( + message.receivedAt, + 'message.receivedAt', + ) + const sentAt = assertOptionalMailboxCanonicalIsoTimestamp( + message.sentAt, + 'message.sentAt', + ) + this.sql.exec( + `INSERT INTO email_messages ( + id, direction, inbox_id, thread_id, sender_identity_id, + from_address, envelope_from, to_addresses_json, cc_addresses_json, + bcc_addresses_json, reply_to_addresses_json, subject, + message_id_header, in_reply_to_header, references_json, headers_json, + auth_results, text_body, html_body, raw_mime_key, raw_size, + processing_status, classification, classification_reason, + provider_message_id, delivery_status, delivery_status_at, error, + received_at, sent_at, created_at, updated_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + direction = excluded.direction, + inbox_id = excluded.inbox_id, + thread_id = excluded.thread_id, + sender_identity_id = excluded.sender_identity_id, + from_address = excluded.from_address, + envelope_from = excluded.envelope_from, + to_addresses_json = excluded.to_addresses_json, + cc_addresses_json = excluded.cc_addresses_json, + bcc_addresses_json = excluded.bcc_addresses_json, + reply_to_addresses_json = excluded.reply_to_addresses_json, + subject = excluded.subject, + message_id_header = excluded.message_id_header, + in_reply_to_header = excluded.in_reply_to_header, + references_json = excluded.references_json, + headers_json = excluded.headers_json, + auth_results = excluded.auth_results, + text_body = excluded.text_body, + html_body = excluded.html_body, + raw_mime_key = excluded.raw_mime_key, + raw_size = excluded.raw_size, + processing_status = excluded.processing_status, + classification = excluded.classification, + classification_reason = excluded.classification_reason, + provider_message_id = excluded.provider_message_id, + delivery_status = CASE + WHEN excluded.delivery_status_at IS NULL + THEN email_messages.delivery_status + WHEN email_messages.delivery_status_at IS NULL + THEN excluded.delivery_status + WHEN excluded.delivery_status_at >= email_messages.delivery_status_at + THEN excluded.delivery_status + ELSE email_messages.delivery_status + END, + delivery_status_at = CASE + WHEN excluded.delivery_status_at IS NULL + THEN email_messages.delivery_status_at + WHEN email_messages.delivery_status_at IS NULL + THEN excluded.delivery_status_at + WHEN excluded.delivery_status_at >= email_messages.delivery_status_at + THEN excluded.delivery_status_at + ELSE email_messages.delivery_status_at + END, + error = excluded.error, + received_at = excluded.received_at, + sent_at = excluded.sent_at, + updated_at = excluded.updated_at`, + id, + direction, + message.inboxId ?? null, + message.threadId ?? null, + message.senderIdentityId ?? null, + message.fromAddress ?? '', + message.envelopeFrom ?? null, + JSON.stringify(message.toAddresses ?? []), + JSON.stringify(message.ccAddresses ?? []), + JSON.stringify(message.bccAddresses ?? []), + JSON.stringify(message.replyToAddresses ?? []), + message.subject ?? '', + message.messageIdHeader ?? null, + message.inReplyToHeader ?? null, + JSON.stringify(message.references ?? []), + message.headers ? JSON.stringify(message.headers) : '{}', + message.authResults ?? null, + boundedEmailBody(message.textBody), + boundedEmailBody(message.htmlBody), + message.rawMimeKey ?? null, + message.rawSize ?? 0, + processingStatus, + classification, + message.classificationReason ?? null, + message.providerMessageId ?? null, + deliveryStatus, + deliveryStatusAt, + message.error ?? null, + receivedAt, + sentAt, + createdAt, + updatedAt, + ) + } + + replaceAttachmentsForMessage( + messageId: string, + attachments: Array, + ) { + this.sql.exec( + `DELETE FROM email_attachments WHERE message_id = ?`, + messageId, + ) + for (const attachment of attachments) { + const id = assertMailboxNonEmptyString(attachment.id, 'attachment.id') + const attachmentMessageId = assertMailboxNonEmptyString( + attachment.messageId, + 'attachment.messageId', + ) + if (attachmentMessageId !== messageId) { + throw new Error( + 'Mailbox attachment.messageId must match the mirrored message id.', + ) + } + const storageKind = assertMailboxStorageKind( + String(attachment.storageKind), + ) + const createdAt = assertMailboxCanonicalIsoTimestamp( + attachment.createdAt?.trim() || mailboxNowIso(), + 'attachment.createdAt', + ) + this.sql.exec( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, content_id, disposition, + size, storage_kind, storage_key, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + id, + attachmentMessageId, + attachment.filename ?? null, + attachment.contentType ?? 'application/octet-stream', + attachment.contentId ?? null, + attachment.disposition ?? null, + Math.max(0, Math.trunc(Number(attachment.size) || 0)), + storageKind, + attachment.storageKey ?? null, + createdAt, + ) + } + } + + findDeliveryEventByProviderEventId( + providerEventId: string, + ): MailboxDeliveryEventRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE provider_event_id = ? + LIMIT 1`, + providerEventId, + ) + .toArray()[0] + return row ? mapMailboxDeliveryEventRow(row) : null + } + + writeDeliveryEventRow(event: MailboxDeliveryEventInput): boolean { + const id = assertMailboxNonEmptyString(event.id, 'event.id') + const eventType = assertMailboxDeliveryEventType(event.eventType) + const providerEventId = event.providerEventId ?? null + if (providerEventId) { + const existingByProvider = + this.findDeliveryEventByProviderEventId(providerEventId) + if (existingByProvider && existingByProvider.id !== id) { + return false + } + } + const existed = this.sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + const createdAt = assertMailboxCanonicalIsoTimestamp( + event.createdAt?.trim() || mailboxNowIso(), + 'event.createdAt', + ) + const detailJson = + typeof event.detailJson === 'string' && event.detailJson.length > 0 + ? event.detailJson + : '{}' + const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 + const state = + event.state == null + ? null + : assertMailboxInboundDeliveryState(event.state) + const subscriptionEffectState = + event.subscriptionEffectState == null + ? null + : assertMailboxSubscriptionEffectState(event.subscriptionEffectState) + + this.sql.exec( + `INSERT INTO email_delivery_events ( + id, message_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, state, fingerprint, + storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, + cleanup_retry_at, expected_attachment_count, finalization_token, + reconcile_after, dedupe_expires_at, + usage_effect_recorded_at, usage_effect_suppressed_at, usage_started_at, + usage_month, usage_bytes, usage_duration_ms, + usage_effect_retry_at, usage_effect_lease, usage_effect_lease_at, + subscription_effect_state, subscription_effect_lease, + subscription_effect_lease_at, subscription_effect_retry_at, + subscription_effect_attempt_count, subscription_effect_dead_letter_at, + subscription_effect_last_error, created_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + message_id = excluded.message_id, + inbox_id = excluded.inbox_id, + event_type = excluded.event_type, + provider = excluded.provider, + provider_message_id = excluded.provider_message_id, + provider_event_id = excluded.provider_event_id, + detail_json = excluded.detail_json, + needs_effect_reconcile = excluded.needs_effect_reconcile, + state = excluded.state, + fingerprint = excluded.fingerprint, + storage_lease = excluded.storage_lease, + storage_lease_at = excluded.storage_lease_at, + cleanup_lease = excluded.cleanup_lease, + cleanup_lease_at = excluded.cleanup_lease_at, + cleanup_retry_at = excluded.cleanup_retry_at, + expected_attachment_count = excluded.expected_attachment_count, + finalization_token = excluded.finalization_token, + reconcile_after = excluded.reconcile_after, + dedupe_expires_at = excluded.dedupe_expires_at, + usage_effect_recorded_at = excluded.usage_effect_recorded_at, + usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, + usage_started_at = excluded.usage_started_at, + usage_month = excluded.usage_month, + usage_bytes = excluded.usage_bytes, + usage_duration_ms = excluded.usage_duration_ms, + usage_effect_retry_at = excluded.usage_effect_retry_at, + usage_effect_lease = excluded.usage_effect_lease, + usage_effect_lease_at = excluded.usage_effect_lease_at, + subscription_effect_state = excluded.subscription_effect_state, + subscription_effect_lease = excluded.subscription_effect_lease, + subscription_effect_lease_at = excluded.subscription_effect_lease_at, + subscription_effect_retry_at = excluded.subscription_effect_retry_at, + subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, + subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, + subscription_effect_last_error = excluded.subscription_effect_last_error`, + id, + event.messageId ?? null, + event.inboxId ?? null, + eventType, + event.provider ?? 'kody', + event.providerMessageId ?? null, + providerEventId, + detailJson, + needsEffectReconcile, + state, + event.fingerprint ?? null, + event.storageLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.storageLeaseAt, + 'event.storageLeaseAt', + ), + event.cleanupLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupLeaseAt, + 'event.cleanupLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupRetryAt, + 'event.cleanupRetryAt', + ), + event.expectedAttachmentCount ?? null, + event.finalizationToken ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.reconcileAfter, + 'event.reconcileAfter', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.dedupeExpiresAt, + 'event.dedupeExpiresAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRecordedAt, + 'event.usageEffectRecordedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectSuppressedAt, + 'event.usageEffectSuppressedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageStartedAt, + 'event.usageStartedAt', + ), + event.usageMonth ?? null, + event.usageBytes ?? null, + event.usageDurationMs ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRetryAt, + 'event.usageEffectRetryAt', + ), + event.usageEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectLeaseAt, + 'event.usageEffectLeaseAt', + ), + subscriptionEffectState, + event.subscriptionEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectLeaseAt, + 'event.subscriptionEffectLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectRetryAt, + 'event.subscriptionEffectRetryAt', + ), + event.subscriptionEffectAttemptCount ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectDeadLetterAt, + 'event.subscriptionEffectDeadLetterAt', + ), + event.subscriptionEffectLastError ?? null, + createdAt, + ) + return existed == null + } + + updateLatestDeliveryStatus(input: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + }): boolean { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const deliveryStatus = assertMailboxDeliveryStatus(input.deliveryStatus) + const deliveryStatusAt = assertMailboxCanonicalIsoTimestamp( + input.deliveryStatusAt, + 'deliveryStatusAt', + ) + const cursor = this.sql.exec( + `UPDATE email_messages + SET delivery_status = ?, + delivery_status_at = ?, + updated_at = ? + WHERE id = ? + AND (delivery_status_at IS NULL OR delivery_status_at <= ?)`, + deliveryStatus, + deliveryStatusAt, + mailboxNowIso(), + messageId, + deliveryStatusAt, + ) + return cursor.rowsWritten > 0 + } + + deliveryEventOwnsMessage(eventId: string, messageId: string) { + return ( + this.sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events + WHERE id = ? AND message_id = ? + LIMIT 1`, + eventId, + messageId, + ) + .toArray()[0] != null + ) + } + + getThread(threadId: string): MailboxThreadRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_threads WHERE id = ? LIMIT 1`, + assertMailboxNonEmptyString(threadId, 'threadId'), + ) + .toArray()[0] + return row ? mapMailboxThreadRow(row) : null + } + + getMessage(messageId: string): MailboxMessageRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_messages WHERE id = ? LIMIT 1`, + assertMailboxNonEmptyString(messageId, 'messageId'), + ) + .toArray()[0] + return row ? mapMailboxMessageRow(row) : null + } + + getMessageByMessageIdHeader( + messageIdHeader: string, + ): MailboxMessageRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE message_id_header = ? + LIMIT 1`, + assertMailboxNonEmptyString(messageIdHeader, 'messageIdHeader'), + ) + .toArray()[0] + return row ? mapMailboxMessageRow(row) : null + } + + getOutboundMessageByProviderMessageId( + providerMessageId: string, + ): MailboxMessageRecord | null { + const id = assertMailboxNonEmptyString( + providerMessageId, + 'providerMessageId', + ) + const rows = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE direction = 'outbound' AND provider_message_id = ? + LIMIT 2`, + id, + ) + .toArray() + if (rows.length > 1) { + throw new Error( + `Multiple outbound email messages share provider id: ${id}`, + ) + } + const row = rows[0] + return row ? mapMailboxMessageRow(row) : null + } + + listMessages(input: MailboxListMessagesInput): { + messages: Array + nextCursor: string | null + } { + const limit = normalizeMailboxPageSize(input.limit) + const clauses: Array = ['1 = 1'] + const params: Array = [] + if (input.inboxId) { + clauses.push('inbox_id = ?') + params.push(input.inboxId) + } + if (input.direction) { + clauses.push('direction = ?') + params.push(assertMailboxDirection(input.direction)) + } + if (input.processingStatus) { + clauses.push('processing_status = ?') + params.push(assertMailboxProcessingStatus(input.processingStatus)) + } + if (input.deliveryStatus) { + clauses.push('delivery_status = ?') + params.push(assertMailboxDeliveryStatus(input.deliveryStatus)) + } + if (input.classification) { + clauses.push('classification = ?') + params.push(assertMailboxClassification(input.classification)) + } + if (input.cursor) { + const cursor = decodeMailboxListCursor(input.cursor) + clauses.push('(created_at, id) < (?, ?)') + params.push(cursor.createdAt, cursor.id) + } + params.push(limit + 1) + const rows = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + const last = pageRows[pageRows.length - 1] + return { + messages: pageRows.map(mapMailboxMessageRow), + nextCursor: + hasMore && last + ? encodeMailboxListCursor({ + createdAt: String(last['created_at']), + id: String(last['id']), + }) + : null, + } + } + + searchMessages(input: MailboxSearchMessagesInput): { + messages: Array + } { + if (typeof input.query !== 'string') { + throw new Error('Mailbox search query must be a string.') + } + const limit = normalizeMailboxPageSize(input.limit) + const pattern = `%${escapeLikePattern(input.query.trim().toLowerCase())}%` + const clauses: Array = [ + `( + LOWER(subject) LIKE ? ESCAPE '\\' + OR LOWER(from_address) LIKE ? ESCAPE '\\' + OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' + )`, + ] + const params: Array = [pattern, pattern, pattern] + if (input.inboxId) { + clauses.push('inbox_id = ?') + params.push(input.inboxId) + } + if (input.direction) { + clauses.push('direction = ?') + params.push(assertMailboxDirection(input.direction)) + } + if (input.processingStatus) { + clauses.push('processing_status = ?') + params.push(assertMailboxProcessingStatus(input.processingStatus)) + } + if (input.deliveryStatus) { + clauses.push('delivery_status = ?') + params.push(assertMailboxDeliveryStatus(input.deliveryStatus)) + } + params.push(limit) + const rows = this.sql + .exec>( + `SELECT * FROM email_messages + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + return { messages: rows.map(mapMailboxMessageRow) } + } + + listAttachmentsForMessage(messageId: string): Array { + return this.sql + .exec>( + `SELECT * FROM email_attachments + WHERE message_id = ? + ORDER BY id ASC`, + assertMailboxNonEmptyString(messageId, 'messageId'), + ) + .toArray() + .map(mapMailboxAttachmentRow) + } + + listDeliveryEvents(input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }): Array { + const limit = normalizeMailboxPageSize(input.limit) + const clauses: Array = ['1 = 1'] + const params: Array = [] + if (input.messageId) { + clauses.push('message_id = ?') + params.push(input.messageId) + } + if (input.eventType) { + clauses.push('event_type = ?') + params.push(assertMailboxDeliveryEventType(input.eventType)) + } + params.push(limit) + return this.sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + .map(mapMailboxDeliveryEventRow) + } + + countMailbox(): MailboxCountResult { + const threads = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_threads`) + .one() + const messages = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_messages`) + .one() + const attachments = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_attachments`) + .one() + const deliveryEvents = this.sql + .exec<{ n: number }>(`SELECT COUNT(*) AS n FROM email_delivery_events`) + .one() + return { + threads: Number(threads.n ?? 0) || 0, + messages: Number(messages.n ?? 0) || 0, + attachments: Number(attachments.n ?? 0) || 0, + deliveryEvents: Number(deliveryEvents.n ?? 0) || 0, + } + } + + exportMailbox(input: { + pageSize?: number + startAfter?: string | null + }): MailboxExportResult { + return exportMailboxFromStore(this.sql, input) + } + + listBlobReferences(input: { + pageSize?: number + startAfter?: string | null + }): MailboxBlobReferencePage { + return listMailboxBlobReferences(this.sql, input) + } + + oldestMessageCreatedAt(): string | null { + const row = this.sql + .exec<{ created_at: string }>( + `SELECT created_at FROM email_messages + ORDER BY created_at ASC, id ASC + LIMIT 1`, + ) + .toArray()[0] + return row?.created_at ?? null + } + + oldestDeliveryEventCreatedAt(): string | null { + const row = this.sql + .exec<{ created_at: string }>( + `SELECT created_at FROM email_delivery_events + ORDER BY created_at ASC, id ASC + LIMIT 1`, + ) + .toArray()[0] + return row?.created_at ?? null + } + + listExpiredMessagesForRetention(input: { + cutoff: string + limit: number + }): Array<{ id: string; raw_mime_key: string | null; created_at: string }> { + return this.sql + .exec<{ + id: string + raw_mime_key: string | null + created_at: string + }>( + `SELECT id, raw_mime_key, created_at FROM email_messages + WHERE created_at < ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + input.cutoff, + input.limit, + ) + .toArray() + } + + listAttachmentStorageKeysForMessages( + messageIds: Array, + ): Array<{ message_id: string; storage_key: string }> { + if (messageIds.length === 0) return [] + return this.sql + .exec<{ message_id: string; storage_key: string }>( + `SELECT message_id, storage_key FROM email_attachments + WHERE storage_key IS NOT NULL + AND message_id IN (${messageIds.map(() => '?').join(', ')})`, + ...messageIds, + ) + .toArray() + } + + deleteMessageCascade(messageId: string) { + this.sql.exec( + `DELETE FROM email_attachments WHERE message_id = ?`, + messageId, + ) + this.sql.exec(`DELETE FROM email_messages WHERE id = ?`, messageId) + } + + pruneExpiredDeliveryEvents(input: { cutoff: string; limit: number }) { + this.sql.exec( + `DELETE FROM email_delivery_events + WHERE id IN ( + SELECT id FROM email_delivery_events + WHERE created_at < ? + ORDER BY created_at ASC, id ASC + LIMIT ? + )`, + input.cutoff, + input.limit, + ) + } + + pruneOrphanThreads(limit: number) { + this.sql.exec( + `DELETE FROM email_threads + WHERE id IN ( + SELECT thread.id FROM email_threads thread + WHERE NOT EXISTS ( + SELECT 1 FROM email_messages message + WHERE message.thread_id = thread.id + ) + ORDER BY thread.created_at ASC, thread.id ASC + LIMIT ? + )`, + limit, + ) + } +} diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts new file mode 100644 index 0000000000..a93016b2ab --- /dev/null +++ b/packages/worker/src/email/mailbox-types.ts @@ -0,0 +1,631 @@ +import { + emailClassificationValues, + emailDeliveryEventTypeValues, + emailDeliveryStatusValues, + emailDirectionValues, + emailProcessingStatusValues, + type EmailClassification, + type EmailDeliveryEventType, + type EmailDeliveryStatus, + type EmailDirection, + type EmailProcessingStatus, +} from './types.ts' + +/** + * Mailbox Durable Object wire types, constants, and boundary validators. + * Object identity is ownership — records have no `user_id`. + */ + +export const mailboxMessageRetentionDays = 365 +export const mailboxDeliveryEventRetentionDays = 90 +export const mailboxDefaultPageSize = 100 +export const mailboxMaxPageSize = 500 +export const mailboxRetentionBatchSize = 100 +export const mailboxRetentionAlarmSkewMs = 60_000 +/** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ +export const mailboxRetentionRetryDelayMs = 60 * 60 * 1000 +export const mailboxBlobDeleteMaxKeys = 1000 + +/** Bump when initializeSchema's DDL set changes; warm objects skip DDL. */ +export const mailboxSchemaVersion = 2 +export const mailboxMetaSchemaVersionKey = 'schema_version' + +export const mailboxExportThreadCursorPrefix = 'thread:' +export const mailboxExportMessageCursorPrefix = 'message:' +export const mailboxExportAttachmentCursorPrefix = 'attachment:' +export const mailboxExportDeliveryEventCursorPrefix = 'delivery_event:' + +export const mailboxBlobRefRawMimeCursorPrefix = 'raw_mime:' +export const mailboxBlobRefAttachmentCursorPrefix = 'attachment:' + +export const mailboxStorageKindValues = [ + 'raw-mime', + 'external', + 'unavailable', +] as const +export type MailboxStorageKind = (typeof mailboxStorageKindValues)[number] + +export const mailboxInboundDeliveryStateValues = [ + 'pending', + 'storing', + 'cleaning', + 'received', + 'rejected', + 'orphan-cleaned', +] as const +export type MailboxInboundDeliveryState = + (typeof mailboxInboundDeliveryStateValues)[number] + +export const mailboxSubscriptionEffectStateValues = [ + 'pending', + 'processing', + 'complete', + 'dead-letter', +] as const +export type MailboxSubscriptionEffectState = + (typeof mailboxSubscriptionEffectStateValues)[number] + +/** Canonical UTC ISO-8601 with milliseconds — lexical order matches time order. */ +export const mailboxCanonicalIsoTimestampPattern = + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ + +export type MailboxThreadRecord = { + id: string + inboxId: string | null + subjectNormalized: string | null + rootMessageIdHeader: string | null + lastMessageAt: string + createdAt: string + updatedAt: string +} + +export type MailboxMessageRecord = { + id: string + direction: EmailDirection + inboxId: string | null + threadId: string | null + senderIdentityId: string | null + fromAddress: string | null + envelopeFrom: string | null + toAddresses: Array + ccAddresses: Array + bccAddresses: Array + replyToAddresses: Array + subject: string | null + messageIdHeader: string | null + inReplyToHeader: string | null + references: Array + headers: Record | null + authResults: string | null + textBody: string | null + htmlBody: string | null + rawMimeKey: string | null + rawSize: number | null + processingStatus: EmailProcessingStatus + classification: EmailClassification + classificationReason: string | null + providerMessageId: string | null + deliveryStatus: EmailDeliveryStatus | null + deliveryStatusAt: string | null + error: string | null + receivedAt: string | null + sentAt: string | null + createdAt: string + updatedAt: string +} + +export type MailboxAttachmentRecord = { + id: string + messageId: string + filename: string | null + contentType: string | null + contentId: string | null + disposition: string | null + size: number + storageKind: string + storageKey: string | null + createdAt: string +} + +/** + * Delivery-event row including promoted inbound idempotency / effect columns. + * `detailJson` remains for compatibility with fields not yet promoted. + */ +export type MailboxDeliveryEventRecord = { + id: string + messageId: string | null + inboxId: string | null + eventType: EmailDeliveryEventType + provider: string | null + providerMessageId: string | null + providerEventId: string | null + detailJson: string + needsEffectReconcile: boolean + state: MailboxInboundDeliveryState | null + fingerprint: string | null + storageLease: string | null + storageLeaseAt: string | null + cleanupLease: string | null + cleanupLeaseAt: string | null + cleanupRetryAt: string | null + expectedAttachmentCount: number | null + finalizationToken: string | null + reconcileAfter: string | null + dedupeExpiresAt: string | null + usageEffectRecordedAt: string | null + usageEffectSuppressedAt: string | null + usageStartedAt: string | null + usageMonth: string | null + usageBytes: number | null + usageDurationMs: number | null + usageEffectRetryAt: string | null + usageEffectLease: string | null + usageEffectLeaseAt: string | null + subscriptionEffectState: MailboxSubscriptionEffectState | null + subscriptionEffectLease: string | null + subscriptionEffectLeaseAt: string | null + subscriptionEffectRetryAt: string | null + subscriptionEffectAttemptCount: number | null + subscriptionEffectDeadLetterAt: string | null + subscriptionEffectLastError: string | null + createdAt: string +} + +export type MailboxThreadInput = { + id: string + inboxId?: string | null + subjectNormalized?: string | null + rootMessageIdHeader?: string | null + lastMessageAt: string + createdAt?: string + updatedAt?: string +} + +export type MailboxMessageInput = { + id: string + direction: EmailDirection + inboxId?: string | null + threadId?: string | null + senderIdentityId?: string | null + fromAddress?: string | null + envelopeFrom?: string | null + toAddresses?: Array + ccAddresses?: Array + bccAddresses?: Array + replyToAddresses?: Array + subject?: string | null + messageIdHeader?: string | null + inReplyToHeader?: string | null + references?: Array + headers?: Record | null + authResults?: string | null + textBody?: string | null + htmlBody?: string | null + rawMimeKey?: string | null + rawSize?: number | null + processingStatus: EmailProcessingStatus + classification?: EmailClassification + classificationReason?: string | null + providerMessageId?: string | null + deliveryStatus?: EmailDeliveryStatus | null + deliveryStatusAt?: string | null + error?: string | null + receivedAt?: string | null + sentAt?: string | null + createdAt?: string + updatedAt?: string +} + +export type MailboxAttachmentInput = { + id: string + messageId: string + filename?: string | null + contentType?: string | null + contentId?: string | null + disposition?: string | null + size?: number + storageKind: MailboxStorageKind | string + storageKey?: string | null + createdAt?: string +} + +export type MailboxDeliveryEventInput = { + id: string + messageId?: string | null + inboxId?: string | null + eventType: EmailDeliveryEventType + provider?: string | null + providerMessageId?: string | null + providerEventId?: string | null + detailJson?: string + /** Defaults false — set true only when inbound effects still need reconcile. */ + needsEffectReconcile?: boolean + state?: MailboxInboundDeliveryState | null + fingerprint?: string | null + storageLease?: string | null + storageLeaseAt?: string | null + cleanupLease?: string | null + cleanupLeaseAt?: string | null + cleanupRetryAt?: string | null + expectedAttachmentCount?: number | null + finalizationToken?: string | null + reconcileAfter?: string | null + dedupeExpiresAt?: string | null + usageEffectRecordedAt?: string | null + usageEffectSuppressedAt?: string | null + usageStartedAt?: string | null + usageMonth?: string | null + usageBytes?: number | null + usageDurationMs?: number | null + usageEffectRetryAt?: string | null + usageEffectLease?: string | null + usageEffectLeaseAt?: string | null + subscriptionEffectState?: MailboxSubscriptionEffectState | null + subscriptionEffectLease?: string | null + subscriptionEffectLeaseAt?: string | null + subscriptionEffectRetryAt?: string | null + subscriptionEffectAttemptCount?: number | null + subscriptionEffectDeadLetterAt?: string | null + subscriptionEffectLastError?: string | null + createdAt?: string +} + +export type MailboxExportRow = + | { kind: 'thread'; row: MailboxThreadRecord } + | { kind: 'message'; row: MailboxMessageRecord } + | { kind: 'attachment'; row: MailboxAttachmentRecord } + | { kind: 'delivery_event'; row: MailboxDeliveryEventRecord } + +export type MailboxExportResult = { + rows: Array + nextStartAfter: string | null + truncated: boolean +} + +export type MailboxBlobReference = { + kind: 'raw_mime' | 'attachment' + key: string + messageId: string + attachmentId: string | null +} + +export type MailboxBlobReferencePage = { + references: Array + nextStartAfter: string | null + truncated: boolean +} + +export type MailboxCountResult = { + threads: number + messages: number + attachments: number + deliveryEvents: number +} + +export type MailboxListMessagesInput = { + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + classification?: EmailClassification | null + limit?: number + cursor?: string | null +} + +export type MailboxSearchMessagesInput = { + query: string + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + limit?: number +} + +export type MailboxListCursorPayload = { + createdAt: string + id: string +} + +export type MailboxExportPhase = + | 'thread' + | 'message' + | 'attachment' + | 'delivery_event' + +export type MailboxRpc = { + mirrorMessage: (input: { + thread?: MailboxThreadInput | null + message: MailboxMessageInput + attachments?: Array + }) => Promise<{ ok: true }> + upsertDeliveryEvent: (input: { + event: MailboxDeliveryEventInput + latestDeliveryStatus?: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + } | null + }) => Promise<{ inserted: boolean; updatedLatestStatus: boolean }> + getThread: (input: { + threadId: string + }) => Promise + getMessage: (input: { + messageId: string + }) => Promise + getMessageByMessageIdHeader: (input: { + messageIdHeader: string + }) => Promise + getOutboundMessageByProviderMessageId: (input: { + providerMessageId: string + }) => Promise + listMessages: (input: MailboxListMessagesInput) => Promise<{ + messages: Array + nextCursor: string | null + }> + searchMessages: (input: MailboxSearchMessagesInput) => Promise<{ + messages: Array + }> + listAttachmentsForMessage: (input: { + messageId: string + }) => Promise> + listDeliveryEvents: (input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }) => Promise> + countMailbox: () => Promise + exportMailbox: (input: { + pageSize?: number + startAfter?: string | null + }) => Promise + listBlobReferences: (input: { + pageSize?: number + startAfter?: string | null + }) => Promise + purge: () => Promise<{ ok: true }> +} + +export function mailboxNowIso() { + return new Date().toISOString() +} + +export function normalizeMailboxPageSize(pageSize: number | undefined) { + const requested = + typeof pageSize === 'number' && Number.isFinite(pageSize) + ? Math.trunc(pageSize) + : mailboxDefaultPageSize + return Math.min(Math.max(requested, 1), mailboxMaxPageSize) +} + +export function assertMailboxNonEmptyString( + value: unknown, + label: string, +): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`Mailbox ${label} must be a non-empty string.`) + } + return value +} + +export function assertMailboxCanonicalIsoTimestamp( + value: string, + label: string, +): string { + if (!mailboxCanonicalIsoTimestampPattern.test(value)) { + throw new Error( + `Mailbox ${label} must be a canonical ISO-8601 UTC timestamp (YYYY-MM-DDTHH:mm:ss.sssZ).`, + ) + } + if (!Number.isFinite(Date.parse(value))) { + throw new Error(`Mailbox ${label} is not a valid timestamp.`) + } + return value +} + +export function assertOptionalMailboxCanonicalIsoTimestamp( + value: string | null | undefined, + label: string, +): string | null { + if (value == null) return null + return assertMailboxCanonicalIsoTimestamp(value, label) +} + +export function assertMailboxDirection(value: string): EmailDirection { + if (!(emailDirectionValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox direction must be inbound or outbound; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailDirection +} + +export function assertMailboxProcessingStatus( + value: string, +): EmailProcessingStatus { + if (!(emailProcessingStatusValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox processingStatus is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailProcessingStatus +} + +export function assertMailboxClassification( + value: string, +): EmailClassification { + if (!(emailClassificationValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox classification is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailClassification +} + +export function assertMailboxDeliveryStatus( + value: string, +): EmailDeliveryStatus { + if (!(emailDeliveryStatusValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox deliveryStatus is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailDeliveryStatus +} + +export function assertMailboxDeliveryEventType( + value: string, +): EmailDeliveryEventType { + if ( + !(emailDeliveryEventTypeValues as ReadonlyArray).includes(value) + ) { + throw new Error( + `Mailbox eventType is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as EmailDeliveryEventType +} + +export function assertMailboxStorageKind(value: string): MailboxStorageKind { + if (!(mailboxStorageKindValues as ReadonlyArray).includes(value)) { + throw new Error( + `Mailbox storageKind is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as MailboxStorageKind +} + +export function assertMailboxInboundDeliveryState( + value: string, +): MailboxInboundDeliveryState { + if ( + !(mailboxInboundDeliveryStateValues as ReadonlyArray).includes( + value, + ) + ) { + throw new Error( + `Mailbox delivery event state is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as MailboxInboundDeliveryState +} + +export function assertMailboxSubscriptionEffectState( + value: string, +): MailboxSubscriptionEffectState { + if ( + !(mailboxSubscriptionEffectStateValues as ReadonlyArray).includes( + value, + ) + ) { + throw new Error( + `Mailbox subscriptionEffectState is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as MailboxSubscriptionEffectState +} + +export function encodeMailboxListCursor(payload: MailboxListCursorPayload) { + return btoa(JSON.stringify(payload)) +} + +export function decodeMailboxListCursor( + cursor: string, +): MailboxListCursorPayload { + let parsed: unknown + try { + parsed = JSON.parse(atob(cursor)) as unknown + } catch { + throw new Error('Mailbox list cursor is invalid.') + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Mailbox list cursor is invalid.') + } + const record = parsed as Record + const createdAt = record['createdAt'] + const id = record['id'] + if (typeof createdAt !== 'string' || typeof id !== 'string') { + throw new Error('Mailbox list cursor is invalid.') + } + assertMailboxCanonicalIsoTimestamp(createdAt, 'list cursor createdAt') + assertMailboxNonEmptyString(id, 'list cursor id') + return { createdAt, id } +} + +export function parseMailboxExportCursor(cursor: string | null): { + phase: MailboxExportPhase + startAfterId: string +} { + if (cursor == null || cursor.length === 0) { + return { phase: 'thread', startAfterId: '' } + } + if (cursor.startsWith(mailboxExportThreadCursorPrefix)) { + return { + phase: 'thread', + startAfterId: cursor.slice(mailboxExportThreadCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxExportMessageCursorPrefix)) { + return { + phase: 'message', + startAfterId: cursor.slice(mailboxExportMessageCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxExportAttachmentCursorPrefix)) { + return { + phase: 'attachment', + startAfterId: cursor.slice(mailboxExportAttachmentCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxExportDeliveryEventCursorPrefix)) { + return { + phase: 'delivery_event', + startAfterId: cursor.slice(mailboxExportDeliveryEventCursorPrefix.length), + } + } + throw new Error( + `Mailbox export cursor is invalid; got ${JSON.stringify(cursor)}.`, + ) +} + +export function mailboxExportCursorPrefixForPhase( + phase: MailboxExportPhase, +): string { + switch (phase) { + case 'thread': + return mailboxExportThreadCursorPrefix + case 'message': + return mailboxExportMessageCursorPrefix + case 'attachment': + return mailboxExportAttachmentCursorPrefix + case 'delivery_event': + return mailboxExportDeliveryEventCursorPrefix + default: { + const exhaustive: never = phase + throw new Error(`Unhandled export phase: ${String(exhaustive)}`) + } + } +} + +export function parseMailboxBlobRefCursor(cursor: string | null): { + phase: 'raw_mime' | 'attachment' + startAfterId: string +} { + if (cursor == null || cursor.length === 0) { + return { phase: 'raw_mime', startAfterId: '' } + } + if (cursor.startsWith(mailboxBlobRefAttachmentCursorPrefix)) { + return { + phase: 'attachment', + startAfterId: cursor.slice(mailboxBlobRefAttachmentCursorPrefix.length), + } + } + if (cursor.startsWith(mailboxBlobRefRawMimeCursorPrefix)) { + return { + phase: 'raw_mime', + startAfterId: cursor.slice(mailboxBlobRefRawMimeCursorPrefix.length), + } + } + throw new Error( + `Mailbox blob-reference cursor is invalid; got ${JSON.stringify(cursor)}.`, + ) +} diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 0d50120601..402234b801 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -7,6 +7,7 @@ import { JobManager } from './jobs/manager-do.ts' import { StorageRunner } from './storage-runner.ts' import { RunLog } from './run-records/run-log-do.ts' import { UserMeter } from './entitlements/user-meter-do.ts' +import { Mailbox } from './email/mailbox-do.ts' import { RepoSession } from './repo/repo-session-do.ts' import { PackageRealtimeSession } from '#worker/package-runtime/realtime-session.ts' import { PackageServiceInstance } from '#worker/package-runtime/package-service.ts' @@ -110,6 +111,7 @@ export { StorageRunner, RunLog, UserMeter, + Mailbox, OAuthPurgeCoordinator, } diff --git a/packages/worker/src/user-scoped-durable-object-name.node.test.ts b/packages/worker/src/user-scoped-durable-object-name.node.test.ts index c21190f1f2..04d484a79c 100644 --- a/packages/worker/src/user-scoped-durable-object-name.node.test.ts +++ b/packages/worker/src/user-scoped-durable-object-name.node.test.ts @@ -5,6 +5,7 @@ import { expect, test } from 'vitest' import { durableObjectNameFromParts, jobManagerDurableObjectName, + mailboxDurableObjectName, mcpClientHubDurableObjectName, packageRealtimeSessionDurableObjectName, packageServiceInstanceDurableObjectName, @@ -31,6 +32,10 @@ test('user-scoped Durable Object name helpers preserve frozen idFromName contrac // UserMeter matches RunLog: untrimmed userId is the frozen idFromName. expect(userMeterDurableObjectName(' user-aaa ')).toBe(' user-aaa ') + expect(mailboxDurableObjectName('user-aaa')).toBe('user-aaa') + // Mailbox matches RunLog: untrimmed userId is the frozen idFromName. + expect(mailboxDurableObjectName(' user-aaa ')).toBe(' user-aaa ') + expect(mcpClientHubDurableObjectName(' user-aaa ')).toBe('user-aaa') expect(storageRunnerDurableObjectName('user-aaa', 'job:1')).toBe( diff --git a/packages/worker/src/user-scoped-durable-object-name.ts b/packages/worker/src/user-scoped-durable-object-name.ts index 6fa7e52a20..3b08310189 100644 --- a/packages/worker/src/user-scoped-durable-object-name.ts +++ b/packages/worker/src/user-scoped-durable-object-name.ts @@ -35,6 +35,11 @@ export function userMeterDurableObjectName(userId: string) { return userId } +/** Mailbox — one email-metadata DO per user (untrimmed, like RunLog). */ +export function mailboxDurableObjectName(userId: string) { + return userId +} + /** McpClientHub — one client-hub DO per user (trimmed). */ export function mcpClientHubDurableObjectName(userId: string) { return userId.trim() diff --git a/packages/worker/worker-configuration.d.ts b/packages/worker/worker-configuration.d.ts index 038d2c60c1..e2f197c8e2 100644 --- a/packages/worker/worker-configuration.d.ts +++ b/packages/worker/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --config=packages/worker/wrangler.jsonc --env=production ./packages/worker/worker-configuration.d.ts` (hash: 7b65a7fb6d2acc30d81d3977328dfcba) +// Generated by Wrangler by running `wrangler types --config=packages/worker/wrangler.jsonc --env=production ./packages/worker/worker-configuration.d.ts` (hash: 326802848e273a6c986ec73d8417c98a) // Runtime types generated with workerd@1.20260701.1 2026-04-16 global_fetch_strictly_public,nodejs_compat interface __BaseEnv_Env { OAUTH_KV: KVNamespace; @@ -30,15 +30,6 @@ interface __BaseEnv_Env { DR_EXPORT_ENABLED: "true"; DR_BACKUP_ACCOUNT_ID: "a41d50ecaf0ae0f86dd1824ef6729cb2"; DR_BACKUP_BUCKET_NAME: "kody-production-backups"; - COOKIE_SECRET: string; - SECRET_STORE_KEY: string; - APP_BASE_URL: string; - GITHUB_CLIENT_ID: string; - GITHUB_CLIENT_SECRET: string; - GOOGLE_CLIENT_ID: string; - GOOGLE_CLIENT_SECRET: string; - X_CLIENT_ID: string; - X_CLIENT_SECRET: string; MCP_OBJECT: DurableObjectNamespace; REMOTE_CONNECTOR_SESSION: DurableObjectNamespace; MCP_CLIENT_HUB: DurableObjectNamespace; @@ -47,6 +38,7 @@ interface __BaseEnv_Env { STORAGE_RUNNER: DurableObjectNamespace; RUN_LOG: DurableObjectNamespace; USER_METER: DurableObjectNamespace; + MAILBOX: DurableObjectNamespace; REPO_SESSION: DurableObjectNamespace; PACKAGE_REALTIME_SESSION: DurableObjectNamespace; PACKAGE_SERVICE_INSTANCE: DurableObjectNamespace; @@ -55,7 +47,7 @@ interface __BaseEnv_Env { declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/index"); - durableNamespaces: "MCP" | "JobManager" | "StorageRunner" | "RepoSession" | "PackageRealtimeSession" | "PackageServiceInstance" | "RemoteConnectorSession" | "McpClientHub" | "OAuthPurgeCoordinator" | "RunLog" | "UserMeter"; + durableNamespaces: "MCP" | "JobManager" | "StorageRunner" | "RepoSession" | "PackageRealtimeSession" | "PackageServiceInstance" | "RemoteConnectorSession" | "McpClientHub" | "OAuthPurgeCoordinator" | "RunLog" | "UserMeter" | "Mailbox"; } interface Env extends __BaseEnv_Env {} } @@ -64,7 +56,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } declare module "*.md" { const value: string; diff --git a/packages/worker/wrangler.jsonc b/packages/worker/wrangler.jsonc index 5f286f0d50..c98af70488 100644 --- a/packages/worker/wrangler.jsonc +++ b/packages/worker/wrangler.jsonc @@ -95,6 +95,10 @@ "new_sqlite_classes": ["UserMeter"], "tag": "v21", }, + { + "new_sqlite_classes": ["Mailbox"], + "tag": "v22", + }, ], "observability": { @@ -201,6 +205,10 @@ "class_name": "UserMeter", "name": "USER_METER", }, + { + "class_name": "Mailbox", + "name": "MAILBOX", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", @@ -428,6 +436,10 @@ "class_name": "UserMeter", "name": "USER_METER", }, + { + "class_name": "Mailbox", + "name": "MAILBOX", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", @@ -572,6 +584,10 @@ "class_name": "UserMeter", "name": "USER_METER", }, + { + "class_name": "Mailbox", + "name": "MAILBOX", + }, { "class_name": "RepoSession", "name": "REPO_SESSION", From c21a0fe18dc1b52995ed15f42f68157450bee141 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 06:59:07 +0000 Subject: [PATCH 02/62] fix(email): harden Mailbox storage contracts Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 51 +- .../contributing/architecture/primitives.yaml | 1 + packages/worker/src/email/blob-keys.ts | 24 + .../src/email/mailbox-delivery-events.ts | 308 +++++++++ packages/worker/src/email/mailbox-do.ts | 72 ++- .../src/email/mailbox-do.workers.test.ts | 611 ++++++++++-------- packages/worker/src/email/mailbox-export.ts | 72 ++- packages/worker/src/email/mailbox-mappers.ts | 112 ++-- .../worker/src/email/mailbox-retention.ts | 123 +++- .../email/mailbox-retention.workers.test.ts | 243 +++++++ packages/worker/src/email/mailbox-schema.ts | 64 +- packages/worker/src/email/mailbox-store.ts | 438 ++++++------- .../worker/src/email/mailbox-test-helpers.ts | 122 ++++ packages/worker/src/email/mailbox-types.ts | 34 +- packages/worker/src/email/repo.ts | 25 +- 15 files changed, 1565 insertions(+), 735 deletions(-) create mode 100644 packages/worker/src/email/blob-keys.ts create mode 100644 packages/worker/src/email/mailbox-delivery-events.ts create mode 100644 packages/worker/src/email/mailbox-retention.workers.test.ts create mode 100644 packages/worker/src/email/mailbox-test-helpers.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index c56f08bf91..99179b9d8a 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -456,6 +456,9 @@ transaction, the rebuild snapshots and restores not lost. Runtime types and `ensurePlatformSenderIdentity` provision verified rows only. +- Canonical key builders are `emailRawMimeKey` / `emailAttachmentBlobKey` in + `packages/worker/src/email/blob-keys.ts` (re-exported from + `packages/worker/src/email/repo.ts`). - All reads go through `loadRawMime` in `packages/worker/src/email/repo.ts`, which fetches the blob by `raw_mime_key` only. Attachment content extraction re-parses the resolved MIME from that blob. @@ -582,27 +585,45 @@ SQLite (`packages/worker/src/email/mailbox-do.ts` and siblings under Worker binding is `MAILBOX` (class `Mailbox`; Wrangler SQLite migration tag `v22` via `new_sqlite_classes` in `packages/worker/wrangler.jsonc`). Raw MIME and outbound attachment bytes stay in `EMAIL_BLOBS` R2; the DO stores object -keys, not payload bytes. +keys, not payload bytes. Canonical key builders live in +`packages/worker/src/email/blob-keys.ts`. Naming matches `RunLog`, `JobManager`, and `UserMeter`: one object per untrimmed stable MCP `userId` via `mailboxDurableObjectName(userId)` → `idFromName(userId)` in -`packages/worker/src/user-scoped-durable-object-name.ts`. There is no `user_id` -column inside the DO because the object identity is the user. - -**SQLite ownership** (schema version in `mailbox_meta`, currently -`mailboxSchemaVersion = 2`): - +`packages/worker/src/user-scoped-durable-object-name.ts`. Data rows have no +`user_id` column (object identity is the user). Because a Durable Object cannot +introspect its `idFromName` string, a singleton `mailbox_owner_identity` row +persists `ownerId` on first write and rejects cross-owner RPCs. That persisted +owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` +/ `emailAttachmentBlobKey`). + +**SQLite ownership** (schema version in `mailbox_meta`; ships as +`mailboxSchemaVersion = 1`): + +- `mailbox_owner_identity` — singleton `owner_id` for blob-key validation and + cross-user write rejection - `email_threads`, `email_messages`, `email_attachments`, and `email_delivery_events` for the owning user (same logical names as the D1 tables they will eventually replace) - latest per-message delivery status on `email_messages.delivery_status`, kept separate from send-request `processing_status` +**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` require +`ownerId` and apply equal-or-newer `updatedAt` snapshots (stale snapshots are +ignored, not applied). On `mirrorMessage`, omitting `attachments` preserves +existing attachment metadata; an explicit `attachments: []` clears them. +Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment +`storageKey` values against the canonical builders for that `ownerId`. + **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). -Alarm-driven deletes are strict blob-before-row for `EMAIL_BLOBS` keys (failed -blob deletes skip the row for retry), matching today's D1 retention ordering. +Alarm-driven deletes derive canonical blob keys from `ownerId` + row ids (rather +than trusting stored key strings), then apply strict blob-before-row ordering +for `EMAIL_BLOBS` (failed blob deletes skip the row for retry). After a +retention pass: successful work with expired rows remaining schedules a +near-immediate continuation (`mailboxRetentionContinuationDelayMs`); R2 delete +failures use hourly backoff (`mailboxRetentionRetryDelayMs`). Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state @@ -1103,9 +1124,15 @@ App-owned R2 keys are: prefix, including historical replacements left by earlier cleanup failures. - `email-raw:v1:{userId}/{messageId}` — raw email MIME for the message row that - stores this key in `email_messages.raw_mime_key`. The `userId` prefix is part - of the per-user isolation contract; account deletion deletes a user's blobs by - the keys stored on their rows. + stores this key in `email_messages.raw_mime_key`. Built by `emailRawMimeKey` + in `packages/worker/src/email/blob-keys.ts`. The `userId` prefix is part of + the per-user isolation contract; account deletion removes a user's blobs under + the matching prefix (and any remaining inventoried keys). + +- `email-attachment:v1:{userId}/{messageId}/{attachmentId}` — standalone + attachment bytes (`storage_kind = 'external'`). Built by + `emailAttachmentBlobKey` in `packages/worker/src/email/blob-keys.ts`. Same + per-user prefix isolation and account-deletion coverage as raw MIME. New R2 key prefixes must add corresponding account-deletion coverage or a deliberate retention note, same as KV. All currently registered R2 surfaces use diff --git a/docs/contributing/architecture/primitives.yaml b/docs/contributing/architecture/primitives.yaml index 6a0731c858..fcc9c477e2 100644 --- a/docs/contributing/architecture/primitives.yaml +++ b/docs/contributing/architecture/primitives.yaml @@ -536,6 +536,7 @@ primitives: name: Email blob storage (R2) summary: EMAIL_BLOBS R2 for raw MIME and outbound attachments. code: + - packages/worker/src/email/blob-keys.ts - packages/worker/src/email/repo.ts docs: - docs/contributing/architecture/data-storage.md diff --git a/packages/worker/src/email/blob-keys.ts b/packages/worker/src/email/blob-keys.ts new file mode 100644 index 0000000000..34dd318edf --- /dev/null +++ b/packages/worker/src/email/blob-keys.ts @@ -0,0 +1,24 @@ +/** + * R2 object key for a message's raw MIME payload in the EMAIL_BLOBS + * bucket. The userId prefix is part of the per-user isolation contract: + * account deletion enumerates and deletes a user's blobs by these stored + * keys, and the key can never be forged to point at another user's mail. + * Writers always store this canonical key in `raw_mime_key`. + */ +export function emailRawMimeKey(userId: string, messageId: string) { + return `email-raw:v1:${userId}/${messageId}` +} + +/** + * R2 object key for an attachment stored on its own (storage_kind + * 'external'), used by outbound mail whose bytes never exist as raw MIME. + * The userId prefix follows the same per-user isolation contract as + * emailRawMimeKey. + */ +export function emailAttachmentBlobKey( + userId: string, + messageId: string, + attachmentId: string, +) { + return `email-attachment:v1:${userId}/${messageId}/${attachmentId}` +} diff --git a/packages/worker/src/email/mailbox-delivery-events.ts b/packages/worker/src/email/mailbox-delivery-events.ts new file mode 100644 index 0000000000..8be19c23cc --- /dev/null +++ b/packages/worker/src/email/mailbox-delivery-events.ts @@ -0,0 +1,308 @@ +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxDeliveryEventType, + assertMailboxInboundDeliveryState, + assertMailboxNonEmptyString, + assertMailboxSubscriptionEffectState, + assertOptionalMailboxCanonicalIsoTimestamp, + mailboxNowIso, + normalizeMailboxPageSize, + type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, +} from './mailbox-types.ts' +import { type EmailDeliveryEventType } from './types.ts' +import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' + +export function findDeliveryEventByProviderEventId( + sql: SqlStorage, + providerEventId: string, +): MailboxDeliveryEventRecord | null { + const row = sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE provider_event_id = ? + LIMIT 1`, + providerEventId, + ) + .toArray()[0] + return row ? mapMailboxDeliveryEventRow(row) : null +} + +export function writeMailboxDeliveryEventRow( + sql: SqlStorage, + event: MailboxDeliveryEventInput, +): { inserted: boolean; accepted: boolean } { + const id = assertMailboxNonEmptyString(event.id, 'event.id') + const eventType = assertMailboxDeliveryEventType(event.eventType) + const providerEventId = event.providerEventId ?? null + if (providerEventId) { + const existingByProvider = findDeliveryEventByProviderEventId( + sql, + providerEventId, + ) + if (existingByProvider && existingByProvider.id !== id) { + return { inserted: false, accepted: false } + } + } + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_delivery_events WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + const createdAt = assertMailboxCanonicalIsoTimestamp( + event.createdAt?.trim() || mailboxNowIso(), + 'event.createdAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + event.updatedAt, + 'event.updatedAt', + ) + if (existing && updatedAt < existing.updated_at) { + return { inserted: false, accepted: false } + } + const detailJson = + typeof event.detailJson === 'string' && event.detailJson.length > 0 + ? event.detailJson + : '{}' + const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 + const state = + event.state == null ? null : assertMailboxInboundDeliveryState(event.state) + const subscriptionEffectState = + event.subscriptionEffectState == null + ? null + : assertMailboxSubscriptionEffectState(event.subscriptionEffectState) + + sql.exec( + `INSERT INTO email_delivery_events ( + id, message_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, state, fingerprint, + storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, + cleanup_retry_at, expected_attachment_count, finalization_token, + reconcile_after, dedupe_expires_at, + usage_effect_recorded_at, usage_effect_suppressed_at, usage_started_at, + usage_month, usage_bytes, usage_duration_ms, + usage_effect_retry_at, usage_effect_lease, usage_effect_lease_at, + subscription_effect_state, subscription_effect_lease, + subscription_effect_lease_at, subscription_effect_retry_at, + subscription_effect_attempt_count, subscription_effect_dead_letter_at, + subscription_effect_last_error, created_at, updated_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + message_id = excluded.message_id, + inbox_id = excluded.inbox_id, + event_type = excluded.event_type, + provider = excluded.provider, + provider_message_id = excluded.provider_message_id, + provider_event_id = excluded.provider_event_id, + detail_json = excluded.detail_json, + needs_effect_reconcile = excluded.needs_effect_reconcile, + state = excluded.state, + fingerprint = excluded.fingerprint, + storage_lease = excluded.storage_lease, + storage_lease_at = excluded.storage_lease_at, + cleanup_lease = excluded.cleanup_lease, + cleanup_lease_at = excluded.cleanup_lease_at, + cleanup_retry_at = excluded.cleanup_retry_at, + expected_attachment_count = excluded.expected_attachment_count, + finalization_token = excluded.finalization_token, + reconcile_after = excluded.reconcile_after, + dedupe_expires_at = excluded.dedupe_expires_at, + usage_effect_recorded_at = excluded.usage_effect_recorded_at, + usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, + usage_started_at = excluded.usage_started_at, + usage_month = excluded.usage_month, + usage_bytes = excluded.usage_bytes, + usage_duration_ms = excluded.usage_duration_ms, + usage_effect_retry_at = excluded.usage_effect_retry_at, + usage_effect_lease = excluded.usage_effect_lease, + usage_effect_lease_at = excluded.usage_effect_lease_at, + subscription_effect_state = excluded.subscription_effect_state, + subscription_effect_lease = excluded.subscription_effect_lease, + subscription_effect_lease_at = excluded.subscription_effect_lease_at, + subscription_effect_retry_at = excluded.subscription_effect_retry_at, + subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, + subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, + subscription_effect_last_error = excluded.subscription_effect_last_error, + updated_at = excluded.updated_at + WHERE excluded.updated_at >= email_delivery_events.updated_at`, + id, + event.messageId ?? null, + event.inboxId ?? null, + eventType, + event.provider ?? 'kody', + event.providerMessageId ?? null, + providerEventId, + detailJson, + needsEffectReconcile, + state, + event.fingerprint ?? null, + event.storageLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.storageLeaseAt, + 'event.storageLeaseAt', + ), + event.cleanupLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupLeaseAt, + 'event.cleanupLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.cleanupRetryAt, + 'event.cleanupRetryAt', + ), + event.expectedAttachmentCount ?? null, + event.finalizationToken ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.reconcileAfter, + 'event.reconcileAfter', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.dedupeExpiresAt, + 'event.dedupeExpiresAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRecordedAt, + 'event.usageEffectRecordedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectSuppressedAt, + 'event.usageEffectSuppressedAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageStartedAt, + 'event.usageStartedAt', + ), + event.usageMonth ?? null, + event.usageBytes ?? null, + event.usageDurationMs ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectRetryAt, + 'event.usageEffectRetryAt', + ), + event.usageEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.usageEffectLeaseAt, + 'event.usageEffectLeaseAt', + ), + subscriptionEffectState, + event.subscriptionEffectLease ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectLeaseAt, + 'event.subscriptionEffectLeaseAt', + ), + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectRetryAt, + 'event.subscriptionEffectRetryAt', + ), + event.subscriptionEffectAttemptCount ?? null, + assertOptionalMailboxCanonicalIsoTimestamp( + event.subscriptionEffectDeadLetterAt, + 'event.subscriptionEffectDeadLetterAt', + ), + event.subscriptionEffectLastError ?? null, + createdAt, + updatedAt, + ) + return { inserted: existing == null, accepted: true } +} + +export function deliveryEventOwnsMessage( + sql: SqlStorage, + eventId: string, + messageId: string, +) { + return ( + sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events + WHERE id = ? AND message_id = ? + LIMIT 1`, + eventId, + messageId, + ) + .toArray()[0] != null + ) +} + +export function listMailboxDeliveryEvents( + sql: SqlStorage, + input: { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit?: number + }, +): Array { + const limit = normalizeMailboxPageSize(input.limit) + const clauses: Array = ['1 = 1'] + const params: Array = [] + if (input.messageId) { + clauses.push('message_id = ?') + params.push(input.messageId) + } + if (input.eventType) { + clauses.push('event_type = ?') + params.push(assertMailboxDeliveryEventType(input.eventType)) + } + params.push(limit) + return sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE ${clauses.join(' AND ')} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ...params, + ) + .toArray() + .map(mapMailboxDeliveryEventRow) +} + +export function oldestMailboxDeliveryEventCreatedAt( + sql: SqlStorage, +): string | null { + const row = sql + .exec<{ created_at: string }>( + `SELECT created_at FROM email_delivery_events + ORDER BY created_at ASC, id ASC + LIMIT 1`, + ) + .toArray()[0] + return row?.created_at ?? null +} + +export function pruneExpiredMailboxDeliveryEvents( + sql: SqlStorage, + input: { cutoff: string; limit: number }, +) { + sql.exec( + `DELETE FROM email_delivery_events + WHERE id IN ( + SELECT id FROM email_delivery_events + WHERE created_at < ? + ORDER BY created_at ASC, id ASC + LIMIT ? + )`, + input.cutoff, + input.limit, + ) +} + +export function hasExpiredMailboxDeliveryEvents( + sql: SqlStorage, + cutoff: string, +): boolean { + return ( + sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events + WHERE created_at < ? + LIMIT 1`, + cutoff, + ) + .toArray()[0] != null + ) +} diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index cce457aca3..146a9ac7bf 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -6,6 +6,7 @@ import { type EmailDeliveryStatus, } from './types.ts' import { + computeMailboxRetentionReschedule, enforceMailboxRetention, mailboxRetentionAlarmAtMs, nextMailboxRetentionDueAtMs, @@ -24,6 +25,7 @@ import { type MailboxListMessagesInput, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxRpc, type MailboxSearchMessagesInput, type MailboxThreadInput, type MailboxThreadRecord, @@ -32,13 +34,15 @@ import { /** * Per-owner Mailbox Durable Object: SQLite metadata for threads, messages, * attachments, and delivery events. Object identity is ownership — no - * `user_id` columns. Raw MIME / external attachment bytes stay in - * `EMAIL_BLOBS`; rows retain keys. `system:email` stays in D1 by design. + * `user_id` columns on data rows. Owner binding for blob-key validation is + * a singleton identity row (DO name is not introspectable). Raw MIME / + * external attachment bytes stay in `EMAIL_BLOBS`; rows retain keys. + * `system:email` stays in D1 by design. * * Scaffold only: dual-write / cutover callers are not wired here. */ -class MailboxBase extends DurableObject { +class MailboxBase extends DurableObject implements MailboxRpc { private readonly store: MailboxStore /** * In-isolate cache: once an alarm is scheduled, hot writes skip @@ -66,8 +70,9 @@ class MailboxBase extends DurableObject { * Arming rule (keep this comment accurate — reviewers rely on it): * * - Schedule at most one alarm for the soonest retention due-time. - * - Overdue work uses an hourly retry delay (not now+1s) so persistent R2 - * failures cannot storm alarm wakes; future rows keep due-time scheduling. + * - Overdue work on the write path uses an hourly retry delay (not now+1s). + * - After an alarm: blob failures → hourly backoff; successful pass with + * expired rows remaining → near-immediate continuation; else future due. * - `retentionAlarmArmed` skips getAlarm/setAlarm on hot writes once armed. * - `retentionIdleConfirmed` skips re-evaluation only while no new row has * been written; every write clears it so the next ensure re-arms. @@ -104,43 +109,57 @@ class MailboxBase extends DurableObject { } async alarm(): Promise { - await enforceMailboxRetention({ + const nowMs = Date.now() + const result = await enforceMailboxRetention({ store: this.store, blobs: this.env.EMAIL_BLOBS, }) - const dueAtMs = nextMailboxRetentionDueAtMs(this.store) - const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) - if (alarmAt == null) { + const nextDueAtMs = nextMailboxRetentionDueAtMs(this.store) + const reschedule = computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: result.hadBlobDeleteFailures, + expiredWorkRemaining: result.expiredWorkRemaining, + nextDueAtMs, + }) + if (reschedule.atMs == null) { this.retentionAlarmArmed = false this.retentionIdleConfirmed = true return } - await this.ctx.storage.setAlarm(alarmAt) + await this.ctx.storage.setAlarm(reschedule.atMs) this.retentionAlarmArmed = true this.retentionIdleConfirmed = false } /** Atomic mirror of thread + message + attachments for dual-write. */ async mirrorMessage(input: { + ownerId: string thread?: MailboxThreadInput | null message: MailboxMessageInput attachments?: Array - }): Promise<{ ok: true }> { + }): Promise<{ ok: true; accepted: boolean }> { const message = input.message assertMailboxNonEmptyString(message.id, 'message.id') - const attachments = Array.isArray(input.attachments) - ? input.attachments - : [] + let accepted = false this.ctx.storage.transactionSync(() => { + const ownerId = this.store.assertOwner(input.ownerId) + this.store.validateMessageBlobKeys({ + ownerId, + message, + attachments: input.attachments, + }) if (input.thread) { this.store.upsertThreadRow(input.thread) } - this.store.upsertMessageRow(message) - this.store.replaceAttachmentsForMessage(message.id, attachments) + const messageResult = this.store.upsertMessageRow(message) + accepted = messageResult.accepted + if (accepted && input.attachments !== undefined) { + this.store.replaceAttachmentsForMessage(message.id, input.attachments) + } }) this.markRetentionDirty() await this.ensureRetentionAlarm() - return { ok: true } + return { ok: true, accepted } } /** @@ -148,18 +167,27 @@ class MailboxBase extends DurableObject { * apply a monotonic latest `delivery_status` update on the message. */ async upsertDeliveryEvent(input: { + ownerId: string event: MailboxDeliveryEventInput latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus deliveryStatusAt: string } | null - }): Promise<{ inserted: boolean; updatedLatestStatus: boolean }> { + }): Promise<{ + inserted: boolean + accepted: boolean + updatedLatestStatus: boolean + }> { let inserted = false + let accepted = false let updatedLatestStatus = false this.ctx.storage.transactionSync(() => { - inserted = this.store.writeDeliveryEventRow(input.event) - if (input.latestDeliveryStatus) { + this.store.assertOwner(input.ownerId) + const write = this.store.writeDeliveryEventRow(input.event) + inserted = write.inserted + accepted = write.accepted + if (accepted && input.latestDeliveryStatus) { const eventId = assertMailboxNonEmptyString(input.event.id, 'event.id') const messageId = assertMailboxNonEmptyString( input.latestDeliveryStatus.messageId, @@ -174,7 +202,7 @@ class MailboxBase extends DurableObject { }) this.markRetentionDirty() await this.ensureRetentionAlarm() - return { inserted, updatedLatestStatus } + return { inserted, accepted, updatedLatestStatus } } async getThread(input: { @@ -275,6 +303,7 @@ export type { MailboxRpc } from './mailbox-types.ts' export { mailboxDeliveryEventRetentionDays, mailboxMessageRetentionDays, + mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, type MailboxAttachmentInput, type MailboxAttachmentRecord, @@ -290,3 +319,4 @@ export { type MailboxThreadInput, type MailboxThreadRecord, } from './mailbox-types.ts' +export { computeMailboxRetentionReschedule } from './mailbox-retention.ts' diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index fe6b423dd7..dfb70c369f 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -1,115 +1,18 @@ -import { env } from 'cloudflare:workers' import { runInDurableObject } from 'cloudflare:test' import { expect, test } from 'vitest' -import { consoleWarn } from '#worker/test-support/console-spies.ts' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' -import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' -import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { mapMailboxMessageRow } from './mailbox-mappers.ts' +import { Mailbox } from './mailbox-do.ts' import { - Mailbox, - mailboxDeliveryEventRetentionDays, - mailboxMessageRetentionDays, - mailboxRetentionRetryDelayMs, - type MailboxAttachmentInput, - type MailboxMessageInput, - type MailboxThreadInput, -} from './mailbox-do.ts' - -function mailboxEnv(): MailboxEnv & { MAILBOX: DurableObjectNamespace } { - const mailbox = (env as MailboxEnv).MAILBOX - if (!mailbox) { - throw new Error( - 'MAILBOX Durable Object binding is required for mailbox-do workers tests.', - ) - } - return { MAILBOX: mailbox } -} - -function stubFor(userId: string) { - const { MAILBOX } = mailboxEnv() - return MAILBOX.get(MAILBOX.idFromName(mailboxDurableObjectName(userId))) -} - -function rpcFor(userId: string) { - return mailboxRpc({ env: mailboxEnv(), userId }) -} - -function uniqueUserId(label: string) { - return `mailbox-${label}-${crypto.randomUUID()}` -} - -async function assertMailboxThrows( - pattern: RegExp, - run: () => Promise, -) { - try { - await run() - throw new Error(`Expected mailbox RPC to throw matching ${pattern}`) - } catch (error) { - expect(String(error)).toMatch(pattern) - } -} - -function baseThread( - overrides?: Partial, -): MailboxThreadInput { - const id = overrides?.id ?? crypto.randomUUID() - const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' - return { - id, - inboxId: 'inbox-1', - subjectNormalized: 'hello', - rootMessageIdHeader: ``, - lastMessageAt: at, - createdAt: at, - updatedAt: at, - ...overrides, - } -} - -function baseMessage( - overrides?: Partial, -): MailboxMessageInput { - const id = overrides?.id ?? crypto.randomUUID() - const at = overrides?.createdAt ?? '2026-07-01T12:00:00.000Z' - return { - id, - direction: 'inbound', - inboxId: 'inbox-1', - threadId: null, - fromAddress: 'sender@example.com', - envelopeFrom: 'envelope@example.com', - toAddresses: ['owner@example.com'], - subject: 'Hello mailbox', - messageIdHeader: ``, - rawMimeKey: `email-raw:v1:owner/${id}`, - rawSize: 128, - processingStatus: 'stored', - classification: 'accepted', - receivedAt: at, - createdAt: at, - updatedAt: at, - ...overrides, - } -} - -function baseAttachment( - messageId: string, - overrides?: Partial, -): MailboxAttachmentInput { - const id = overrides?.id ?? crypto.randomUUID() - return { - id, - messageId, - filename: 'note.txt', - contentType: 'text/plain', - size: 12, - storageKind: 'external', - storageKey: `email-attachment:v1:owner/${messageId}/${id}`, - createdAt: '2026-07-01T12:00:00.000Z', - ...overrides, - } -} + assertMailboxThrows, + baseAttachment, + baseMessage, + baseThread, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', async () => { silenceIncidentalRuntimeWarnings() @@ -127,7 +30,8 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', WHERE type = 'table' AND name IN ( 'email_threads', 'email_messages', - 'email_attachments', 'email_delivery_events' + 'email_attachments', 'email_delivery_events', + 'mailbox_owner_identity' ) ORDER BY name ASC`, ) @@ -138,27 +42,34 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', 'email_delivery_events', 'email_messages', 'email_threads', + 'mailbox_owner_identity', ]) expect(await state.storage.getAlarm()).toBeNull() }) const thread = baseThread({ id: 'thread-1' }) - const message = baseMessage({ + const message = baseMessage(ownerA, { id: 'msg-1', threadId: thread.id, subject: 'Project update', fromAddress: 'alice@example.com', }) - const attachment = baseAttachment(message.id, { id: 'att-1' }) + const attachment = baseAttachment(ownerA, message.id, { id: 'att-1' }) await mailboxA.mirrorMessage({ + ownerId: ownerA, thread, message, attachments: [attachment], }) await mailboxA.mirrorMessage({ + ownerId: ownerA, thread, - message: { ...message, subject: 'Project update (edited)' }, + message: { + ...message, + subject: 'Project update (edited)', + updatedAt: '2026-07-01T12:00:01.000Z', + }, attachments: [attachment], }) @@ -168,7 +79,7 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', threadId: thread.id, subject: 'Project update (edited)', fromAddress: 'alice@example.com', - rawMimeKey: message.rawMimeKey, + rawMimeKey: emailRawMimeKey(ownerA, message.id), }) expect(await mailboxA.getThread({ threadId: thread.id })).toMatchObject({ id: thread.id, @@ -198,8 +109,9 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', ).toHaveLength(0) await mailboxB.mirrorMessage({ + ownerId: ownerB, thread: baseThread({ id: 'thread-b' }), - message: baseMessage({ + message: baseMessage(ownerB, { id: 'msg-b', threadId: 'thread-b', subject: 'Other owner', @@ -222,7 +134,8 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', const outboundId = 'msg-out-1' await mailboxA.mirrorMessage({ - message: baseMessage({ + ownerId: ownerA, + message: baseMessage(ownerA, { id: outboundId, direction: 'outbound', subject: 'Sent mail', @@ -236,6 +149,236 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', providerMessageId: 'provider-1', }), ).toMatchObject({ id: outboundId }) + + // Cross-user ownerId and forged blob keys must be rejected. + await runInDurableObject(stubA, async (instance: Mailbox) => { + await assertMailboxThrows(/ownerId mismatch/, () => + instance.mirrorMessage({ + ownerId: ownerB, + message: baseMessage(ownerA, { id: 'cross-user' }), + }), + ) + await assertMailboxThrows(/rawMimeKey must equal/, () => + instance.mirrorMessage({ + ownerId: ownerA, + message: baseMessage(ownerA, { + id: 'bad-key', + rawMimeKey: emailRawMimeKey(ownerB, 'bad-key'), + }), + }), + ) + await assertMailboxThrows(/storageKey must equal/, () => + instance.mirrorMessage({ + ownerId: ownerA, + message: baseMessage(ownerA, { id: 'bad-att-msg' }), + attachments: [ + baseAttachment(ownerA, 'bad-att-msg', { + id: 'bad-att', + storageKey: emailAttachmentBlobKey( + ownerB, + 'bad-att-msg', + 'bad-att', + ), + }), + ], + }), + ) + }) +}) + +test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('stale') + const mailbox = rpcFor(userId) + + const thread = baseThread({ id: 'stale-thread' }) + const message = baseMessage(userId, { + id: 'stale-msg', + threadId: thread.id, + classification: 'accepted', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + const attachment = baseAttachment(userId, message.id, { id: 'stale-att' }) + + await mailbox.mirrorMessage({ + ownerId: userId, + thread, + message, + attachments: [attachment], + }) + + // Omitted attachments leave existing metadata unchanged. + await mailbox.mirrorMessage({ + ownerId: userId, + message: { + ...message, + subject: 'edited without attachments field', + updatedAt: '2026-07-01T12:00:01.000Z', + }, + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: message.id }), + ).toHaveLength(1) + + // Stale snapshot must not regress classification, thread time, or attachments. + const staleMirror = await mailbox.mirrorMessage({ + ownerId: userId, + thread: { + ...thread, + lastMessageAt: '2026-06-01T00:00:00.000Z', + updatedAt: '2026-06-01T00:00:00.000Z', + }, + message: { + ...message, + classification: 'quarantined', + subject: 'stale subject', + updatedAt: '2026-07-01T11:00:00.000Z', + }, + attachments: [], + }) + expect(staleMirror.accepted).toBe(false) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + classification: 'accepted', + subject: 'edited without attachments field', + updatedAt: '2026-07-01T12:00:01.000Z', + }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: thread.lastMessageAt, + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: message.id }), + ).toHaveLength(1) + + // Explicit [] clears only when the message snapshot is accepted. + await mailbox.mirrorMessage({ + ownerId: userId, + message: { + ...message, + subject: 'cleared attachments', + updatedAt: '2026-07-01T12:00:02.000Z', + }, + attachments: [], + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: message.id }), + ).toHaveLength(0) + + const outbound = baseMessage(userId, { + id: 'stale-out', + direction: 'outbound', + providerMessageId: 'prov-stale', + processingStatus: 'sent', + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }) + await mailbox.mirrorMessage({ ownerId: userId, message: outbound }) + await mailbox.mirrorMessage({ + ownerId: userId, + message: { + ...outbound, + deliveryStatus: 'deferred', + deliveryStatusAt: '2026-07-02T09:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }, + }) + expect(await mailbox.getMessage({ messageId: outbound.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + + const first = await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'evt-1', + messageId: outbound.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'prov-stale', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: false, + state: 'received', + fingerprint: 'fp-1', + }, + latestDeliveryStatus: { + messageId: outbound.id, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:30:00.000Z', + }, + }) + expect(first).toEqual({ + inserted: true, + accepted: true, + updatedLatestStatus: true, + }) + + const staleEvent = await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'evt-1', + messageId: outbound.id, + eventType: 'deferred', + provider: 'cloudflare-email', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T09:00:00.000Z', + needsEffectReconcile: true, + state: null, + fingerprint: null, + }, + }) + expect(staleEvent).toEqual({ + inserted: false, + accepted: false, + updatedLatestStatus: false, + }) + const eventRow = ( + await mailbox.listDeliveryEvents({ messageId: outbound.id, limit: 5 }) + ).find((row) => row.id === 'evt-1') + expect(eventRow).toMatchObject({ + eventType: 'delivered', + state: 'received', + fingerprint: 'fp-1', + needsEffectReconcile: false, + updatedAt: '2026-07-02T10:00:00.000Z', + }) + + const duplicate = await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'evt-1-dup', + messageId: outbound.id, + eventType: 'delivered', + provider: 'cloudflare-email', + providerEventId: 'provider-event-1', + createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', + }, + }) + expect(duplicate).toEqual({ + inserted: false, + accepted: false, + updatedLatestStatus: false, + }) + + expect(() => + mapMailboxMessageRow({ + id: 'corrupt', + direction: 'bogus', + processing_status: 'stored', + classification: 'accepted', + to_addresses_json: '[]', + cc_addresses_json: '[]', + bcc_addresses_json: '[]', + reply_to_addresses_json: '[]', + references_json: '[]', + headers_json: '{}', + created_at: '2026-07-01T12:00:00.000Z', + updated_at: '2026-07-01T12:00:00.000Z', + }), + ).toThrow(/persisted direction is invalid/) }) test('Mailbox delivery status, promoted inbound fields, export paging, and cursor rejection', async () => { @@ -244,34 +387,47 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const mailbox = rpcFor(userId) const thread = baseThread({ id: 'export-thread' }) - const message = baseMessage({ + const message = baseMessage(userId, { id: 'export-msg', threadId: thread.id, direction: 'outbound', providerMessageId: 'prov-export', processingStatus: 'sent', - rawMimeKey: 'email-raw:v1:owner/export-msg', + rawMimeKey: null, deliveryStatus: 'delivered', deliveryStatusAt: '2026-07-02T10:00:00.000Z', }) - const attachment = baseAttachment(message.id, { + const attachment = baseAttachment(userId, message.id, { id: 'export-att', - storageKey: 'email-attachment:v1:owner/export-msg/export-att', + }) + const inboundBlobMessage = baseMessage(userId, { + id: 'export-inbound', + subject: 'inbound for blobs', + }) + const inboundAttachment = baseAttachment(userId, inboundBlobMessage.id, { + id: 'export-inbound-att', }) await mailbox.mirrorMessage({ + ownerId: userId, thread, message, attachments: [attachment], }) + await mailbox.mirrorMessage({ + ownerId: userId, + message: inboundBlobMessage, + attachments: [inboundAttachment], + }) // Stale dual-write replay must not regress a newer delivery status. await mailbox.mirrorMessage({ + ownerId: userId, message: { ...message, deliveryStatus: 'deferred', deliveryStatusAt: '2026-07-02T09:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', }, - attachments: [attachment], }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ deliveryStatus: 'delivered', @@ -280,12 +436,13 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso // Equal timestamps may update (matches D1 <= semantics). await mailbox.mirrorMessage({ + ownerId: userId, message: { ...message, deliveryStatus: 'complained', deliveryStatusAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:02.000Z', }, - attachments: [attachment], }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ deliveryStatus: 'complained', @@ -293,6 +450,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso }) const first = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-1', messageId: message.id, @@ -301,6 +459,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerMessageId: 'prov-export', providerEventId: 'provider-event-1', createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', needsEffectReconcile: false, }, latestDeliveryStatus: { @@ -309,9 +468,14 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryStatusAt: '2026-07-02T10:30:00.000Z', }, }) - expect(first).toEqual({ inserted: true, updatedLatestStatus: true }) + expect(first).toEqual({ + inserted: true, + accepted: true, + updatedLatestStatus: true, + }) const inbound = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'inbound-delivery-1', messageId: 'inbound-msg-1', @@ -319,6 +483,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso eventType: 'received', provider: 'cloudflare-email-routing', createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', needsEffectReconcile: true, state: 'received', fingerprint: 'fp-abc', @@ -356,6 +521,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso usageBytes: 2048, subscriptionEffectState: 'pending', subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', }) expect(JSON.parse(inboundRow!.detailJson)).toMatchObject({ recipient: 'owner@example.com', @@ -363,12 +529,14 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso // Default needsEffectReconcile is false unless explicitly true. await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-default-reconcile', messageId: message.id, eventType: 'sent', provider: 'kody', createdAt: '2026-07-02T11:30:00.000Z', + updatedAt: '2026-07-02T11:30:00.000Z', }, }) const defaultReconcile = ( @@ -377,6 +545,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso expect(defaultReconcile?.needsEffectReconcile).toBe(false) const duplicate = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-1-dup', messageId: message.id, @@ -385,6 +554,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerMessageId: 'prov-export', providerEventId: 'provider-event-1', createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', }, latestDeliveryStatus: { messageId: message.id, @@ -393,8 +563,10 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso }, }) expect(duplicate.inserted).toBe(false) + expect(duplicate.accepted).toBe(false) const stale = await mailbox.upsertDeliveryEvent({ + ownerId: userId, event: { id: 'evt-2', messageId: message.id, @@ -403,6 +575,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerMessageId: 'prov-export', providerEventId: 'provider-event-2', createdAt: '2026-07-02T09:00:00.000Z', + updatedAt: '2026-07-02T09:00:00.000Z', }, latestDeliveryStatus: { messageId: message.id, @@ -410,14 +583,16 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryStatusAt: '2026-07-02T09:00:00.000Z', }, }) - expect(stale).toEqual({ inserted: true, updatedLatestStatus: false }) + expect(stale).toEqual({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ deliveryStatus: 'delivered', deliveryStatusAt: '2026-07-02T10:30:00.000Z', }) - // Assert validation failures inside the DO isolate so remote RPC error - // plumbing does not surface uncaught worker exceptions. const stub = stubFor(userId) await runInDurableObject(stub, async (instance: Mailbox) => { await assertMailboxThrows(/export cursor is invalid/, () => @@ -431,9 +606,11 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso ) await assertMailboxThrows(/canonical ISO-8601/, () => instance.mirrorMessage({ - message: baseMessage({ + ownerId: userId, + message: baseMessage(userId, { id: 'bad-iso', createdAt: '2026-07-01T12:00:00Z', + updatedAt: '2026-07-01T12:00:00Z', }), }), ) @@ -441,7 +618,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const kinds: Array = [] let startAfter: string | null = null - for (let page = 0; page < 30; page += 1) { + for (let page = 0; page < 40; page += 1) { const exported = await mailbox.exportMailbox({ pageSize: 1, startAfter, @@ -453,10 +630,10 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso expect(row.row.id).toBe(thread.id) break case 'message': - expect(row.row.id).toBe(message.id) + expect(['export-msg', 'export-inbound']).toContain(row.row.id) break case 'attachment': - expect(row.row.id).toBe(attachment.id) + expect(['export-att', 'export-inbound-att']).toContain(row.row.id) break case 'delivery_event': expect(typeof row.row.id).toBe('string') @@ -472,13 +649,14 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso startAfter = exported.nextStartAfter } expect(kinds.filter((kind) => kind === 'thread')).toHaveLength(1) - expect(kinds.filter((kind) => kind === 'message')).toHaveLength(1) - expect(kinds.filter((kind) => kind === 'attachment')).toHaveLength(1) + expect(kinds.filter((kind) => kind === 'message')).toHaveLength(2) + expect(kinds.filter((kind) => kind === 'attachment')).toHaveLength(2) expect( kinds.filter((kind) => kind === 'delivery_event').length, ).toBeGreaterThanOrEqual(3) const blobKinds: Array = [] + const blobKeys: Array = [] let blobCursor: string | null = null for (let page = 0; page < 10; page += 1) { const pageResult = await mailbox.listBlobReferences({ @@ -487,19 +665,31 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso }) for (const reference of pageResult.references) { blobKinds.push(reference.kind) + blobKeys.push(reference.key) if (reference.kind === 'raw_mime') { - expect(reference.key).toBe(message.rawMimeKey) - expect(reference.messageId).toBe(message.id) + expect(reference.key).toBe( + emailRawMimeKey(userId, inboundBlobMessage.id), + ) + expect(reference.messageId).toBe(inboundBlobMessage.id) } else { - expect(reference.key).toBe(attachment.storageKey) - expect(reference.attachmentId).toBe(attachment.id) + expect(reference.key).toBe( + emailAttachmentBlobKey( + userId, + reference.messageId, + reference.attachmentId!, + ), + ) } } if (!pageResult.truncated) break expect(pageResult.nextStartAfter).not.toBe(blobCursor) blobCursor = pageResult.nextStartAfter } - expect(blobKinds).toEqual(['raw_mime', 'attachment']) + expect(blobKinds[0]).toBe('raw_mime') + expect(blobKinds.filter((kind) => kind === 'attachment').length).toBe(2) + expect(blobKeys).toContain( + emailAttachmentBlobKey(userId, message.id, attachment.id), + ) await mailbox.purge() expect(await mailbox.countMailbox()).toEqual({ @@ -509,152 +699,3 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryEvents: 0, }) }) - -test('Mailbox retention deletes R2 blobs before metadata and backs off on failure', async () => { - silenceIncidentalRuntimeWarnings() - consoleWarn.mockImplementation((...args: Array) => { - const message = String(args[0] ?? '') - if (message.includes('mailbox-retention-blob-delete-failed')) return - }) - - const userId = uniqueUserId('retention') - const mailbox = rpcFor(userId) - const stub = stubFor(userId) - - const oldMessageAt = new Date( - Date.now() - (mailboxMessageRetentionDays + 3) * 24 * 60 * 60 * 1000, - ).toISOString() - const oldEventAt = new Date( - Date.now() - (mailboxDeliveryEventRetentionDays + 3) * 24 * 60 * 60 * 1000, - ).toISOString() - const freshAt = new Date().toISOString() - - const keepMessage = baseMessage({ - id: 'keep-msg', - subject: 'fresh', - createdAt: freshAt, - updatedAt: freshAt, - rawMimeKey: 'email-raw:v1:owner/keep-msg', - }) - const dropMessage = baseMessage({ - id: 'drop-msg', - threadId: 'drop-thread', - subject: 'old', - createdAt: oldMessageAt, - updatedAt: oldMessageAt, - rawMimeKey: 'email-raw:v1:owner/drop-msg', - }) - const failMessage = baseMessage({ - id: 'fail-msg', - subject: 'fail-old', - createdAt: oldMessageAt, - updatedAt: oldMessageAt, - rawMimeKey: 'email-raw:v1:owner/fail-msg', - }) - const dropAttachment = baseAttachment(dropMessage.id, { - id: 'drop-att', - storageKey: 'email-attachment:v1:owner/drop-msg/drop-att', - createdAt: oldMessageAt, - }) - - await mailbox.mirrorMessage({ - thread: baseThread({ - id: 'drop-thread', - lastMessageAt: oldMessageAt, - createdAt: oldMessageAt, - updatedAt: oldMessageAt, - }), - message: dropMessage, - attachments: [dropAttachment], - }) - await mailbox.mirrorMessage({ message: keepMessage }) - await mailbox.mirrorMessage({ message: failMessage }) - await mailbox.upsertDeliveryEvent({ - event: { - id: 'old-event', - messageId: keepMessage.id, - eventType: 'received', - provider: 'kody', - createdAt: oldEventAt, - needsEffectReconcile: false, - }, - }) - await mailbox.upsertDeliveryEvent({ - event: { - id: 'fresh-event', - messageId: keepMessage.id, - eventType: 'received', - provider: 'kody', - createdAt: freshAt, - needsEffectReconcile: false, - }, - }) - - await env.EMAIL_BLOBS.put(dropMessage.rawMimeKey!, 'drop-raw') - await env.EMAIL_BLOBS.put(dropAttachment.storageKey!, 'drop-att') - await env.EMAIL_BLOBS.put(failMessage.rawMimeKey!, 'fail-raw') - await env.EMAIL_BLOBS.put(keepMessage.rawMimeKey!, 'keep-raw') - - const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) - env.EMAIL_BLOBS.delete = (async (keys: string | Array) => { - const list = Array.isArray(keys) ? keys : [keys] - if (list.includes(failMessage.rawMimeKey!)) { - throw new Error('simulated R2 delete failure') - } - return await originalDelete(keys) - }) as typeof env.EMAIL_BLOBS.delete - - try { - const alarmBefore = Date.now() - const alarmAfterFailure = await runInDurableObject( - stub, - async (instance: Mailbox, state) => { - expect(instance).toBeInstanceOf(Mailbox) - await state.storage.deleteAlarm() - await instance.alarm() - return await state.storage.getAlarm() - }, - ) - // Overdue retained work (failed R2 delete) must retry with hourly backoff, - // not every second. - expect(alarmAfterFailure).toBeTypeOf('number') - expect(alarmAfterFailure).toBeGreaterThanOrEqual( - alarmBefore + mailboxRetentionRetryDelayMs - 5_000, - ) - expect(alarmAfterFailure).toBeLessThanOrEqual( - alarmBefore + mailboxRetentionRetryDelayMs + 5_000, - ) - - expect(await env.EMAIL_BLOBS.get(dropMessage.rawMimeKey!)).toBeNull() - expect(await env.EMAIL_BLOBS.get(dropAttachment.storageKey!)).toBeNull() - expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).not.toBeNull() - expect(await env.EMAIL_BLOBS.get(keepMessage.rawMimeKey!)).not.toBeNull() - - expect(await mailbox.getMessage({ messageId: dropMessage.id })).toBeNull() - expect( - await mailbox.listAttachmentsForMessage({ messageId: dropMessage.id }), - ).toHaveLength(0) - expect(await mailbox.getThread({ threadId: 'drop-thread' })).toBeNull() - expect( - await mailbox.getMessage({ messageId: failMessage.id }), - ).toMatchObject({ - id: failMessage.id, - rawMimeKey: failMessage.rawMimeKey, - }) - expect( - await mailbox.getMessage({ messageId: keepMessage.id }), - ).toMatchObject({ - id: keepMessage.id, - }) - const events = await mailbox.listDeliveryEvents({ limit: 10 }) - expect(events.map((event) => event.id)).toEqual(['fresh-event']) - } finally { - env.EMAIL_BLOBS.delete = originalDelete - } - - await runInDurableObject(stub, async (instance: Mailbox) => { - await instance.alarm() - }) - expect(await env.EMAIL_BLOBS.get(failMessage.rawMimeKey!)).toBeNull() - expect(await mailbox.getMessage({ messageId: failMessage.id })).toBeNull() -}) diff --git a/packages/worker/src/email/mailbox-export.ts b/packages/worker/src/email/mailbox-export.ts index e2202c6999..a0dc656c72 100644 --- a/packages/worker/src/email/mailbox-export.ts +++ b/packages/worker/src/email/mailbox-export.ts @@ -1,3 +1,4 @@ +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { mailboxBlobRefAttachmentCursorPrefix, mailboxBlobRefRawMimeCursorPrefix, @@ -183,13 +184,23 @@ function exportPhasePage( } } +/** + * List owner-safe blob references. Inbound raw MIME keys are always derived + * from the canonical helper (even when the stored column is null). External + * attachment keys are included only when they match the canonical helper. + */ export function listMailboxBlobReferences( sql: SqlStorage, input: { + ownerId: string | null pageSize?: number startAfter?: string | null }, ): MailboxBlobReferencePage { + if (input.ownerId == null) { + return { references: [], nextStartAfter: null, truncated: false } + } + const ownerId = input.ownerId const pageSize = normalizeMailboxPageSize(input.pageSize) const startAfterRaw = typeof input.startAfter === 'string' && input.startAfter.length > 0 @@ -201,9 +212,9 @@ export function listMailboxBlobReferences( if (phase === 'raw_mime') { const rows = sql - .exec<{ id: string; raw_mime_key: string }>( - `SELECT id, raw_mime_key FROM email_messages - WHERE raw_mime_key IS NOT NULL AND id > ? + .exec<{ id: string }>( + `SELECT id FROM email_messages + WHERE direction = 'inbound' AND id > ? ORDER BY id ASC LIMIT ?`, startAfterId, @@ -215,7 +226,7 @@ export function listMailboxBlobReferences( for (const row of pageRows) { references.push({ kind: 'raw_mime', - key: row.raw_mime_key, + key: emailRawMimeKey(ownerId, row.id), messageId: row.id, attachmentId: null, }) @@ -237,6 +248,7 @@ export function listMailboxBlobReferences( } } const attachmentPage = listAttachmentBlobReferences(sql, { + ownerId, startAfterId: '', limit: remaining, }) @@ -249,6 +261,7 @@ export function listMailboxBlobReferences( } return listAttachmentBlobReferences(sql, { + ownerId, startAfterId, limit: remaining, }) @@ -257,6 +270,7 @@ export function listMailboxBlobReferences( function listAttachmentBlobReferences( sql: SqlStorage, input: { + ownerId: string startAfterId: string limit: number }, @@ -264,30 +278,48 @@ function listAttachmentBlobReferences( if (input.limit <= 0) { return { references: [], nextStartAfter: null, truncated: false } } + // Scan past non-canonical keys so callers never receive forgeable refs. + const scanLimit = Math.max(input.limit * 4, 32) const rows = sql - .exec<{ id: string; message_id: string; storage_key: string }>( + .exec<{ id: string; message_id: string; storage_key: string | null }>( `SELECT id, message_id, storage_key FROM email_attachments - WHERE storage_key IS NOT NULL AND id > ? + WHERE storage_kind = 'external' AND id > ? ORDER BY id ASC LIMIT ?`, input.startAfterId, - input.limit + 1, + scanLimit, ) .toArray() - const truncated = rows.length > input.limit - const pageRows = truncated ? rows.slice(0, input.limit) : rows - const last = pageRows[pageRows.length - 1] - return { - references: pageRows.map((row) => ({ - kind: 'attachment' as const, - key: row.storage_key, + const references: Array = [] + let lastScannedId = input.startAfterId + for (const row of rows) { + lastScannedId = row.id + const expected = emailAttachmentBlobKey( + input.ownerId, + row.message_id, + row.id, + ) + if (row.storage_key !== expected) continue + references.push({ + kind: 'attachment', + key: expected, messageId: row.message_id, attachmentId: row.id, - })), - nextStartAfter: - truncated && last - ? `${mailboxBlobRefAttachmentCursorPrefix}${last.id}` - : null, - truncated, + }) + if (references.length >= input.limit) { + return { + references, + nextStartAfter: `${mailboxBlobRefAttachmentCursorPrefix}${row.id}`, + truncated: true, + } + } + } + const maybeMore = rows.length >= scanLimit + return { + references, + nextStartAfter: maybeMore + ? `${mailboxBlobRefAttachmentCursorPrefix}${lastScannedId}` + : null, + truncated: maybeMore, } } diff --git a/packages/worker/src/email/mailbox-mappers.ts b/packages/worker/src/email/mailbox-mappers.ts index 9934469b5c..f9cb3c9f28 100644 --- a/packages/worker/src/email/mailbox-mappers.ts +++ b/packages/worker/src/email/mailbox-mappers.ts @@ -18,11 +18,13 @@ import { } from './types.ts' import { mailboxInboundDeliveryStateValues, + mailboxStorageKindValues, mailboxSubscriptionEffectStateValues, type MailboxAttachmentRecord, type MailboxDeliveryEventRecord, type MailboxInboundDeliveryState, type MailboxMessageRecord, + type MailboxStorageKind, type MailboxSubscriptionEffectState, type MailboxThreadRecord, } from './mailbox-types.ts' @@ -60,26 +62,27 @@ export function escapeLikePattern(value: string) { return value.replaceAll(/[\\%_]/g, (character) => `\\${character}`) } -function mapInboundState(value: unknown): MailboxInboundDeliveryState | null { - if (value == null) return null - const state = String(value) - return (mailboxInboundDeliveryStateValues as ReadonlyArray).includes( - state, - ) - ? (state as MailboxInboundDeliveryState) - : null +function requirePersistedEnum( + values: ReadonlyArray, + value: unknown, + label: string, +): T { + const raw = String(value ?? '') + if (!(values as ReadonlyArray).includes(raw)) { + throw new Error( + `Mailbox persisted ${label} is invalid; got ${JSON.stringify(value)}.`, + ) + } + return raw as T } -function mapSubscriptionEffectState( +function requireOptionalPersistedEnum( + values: ReadonlyArray, value: unknown, -): MailboxSubscriptionEffectState | null { + label: string, +): T | null { if (value == null) return null - const state = String(value) - return ( - mailboxSubscriptionEffectStateValues as ReadonlyArray - ).includes(state) - ? (state as MailboxSubscriptionEffectState) - : null + return requirePersistedEnum(values, value, label) } export function mapMailboxThreadRow( @@ -105,18 +108,13 @@ export function mapMailboxThreadRow( export function mapMailboxMessageRow( row: Record, ): MailboxMessageRecord { - const direction = String(row['direction'] ?? '') - const processingStatus = String(row['processing_status'] ?? '') - const classificationRaw = String(row['classification'] ?? 'accepted') - const deliveryStatusRaw = - row['delivery_status'] == null ? null : String(row['delivery_status']) return { id: String(row['id']), - direction: (emailDirectionValues as ReadonlyArray).includes( - direction, - ) - ? (direction as EmailDirection) - : 'inbound', + direction: requirePersistedEnum( + emailDirectionValues, + row['direction'], + 'direction', + ) as EmailDirection, inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), threadId: row['thread_id'] == null ? null : String(row['thread_id']), senderIdentityId: @@ -153,16 +151,16 @@ export function mapMailboxMessageRow( rawMimeKey: row['raw_mime_key'] == null ? null : String(row['raw_mime_key']), rawSize: row['raw_size'] == null ? null : Number(row['raw_size']) || 0, - processingStatus: ( - emailProcessingStatusValues as ReadonlyArray - ).includes(processingStatus) - ? (processingStatus as EmailProcessingStatus) - : 'stored', - classification: ( - emailClassificationValues as ReadonlyArray - ).includes(classificationRaw) - ? (classificationRaw as EmailClassification) - : 'accepted', + processingStatus: requirePersistedEnum( + emailProcessingStatusValues, + row['processing_status'], + 'processingStatus', + ) as EmailProcessingStatus, + classification: requirePersistedEnum( + emailClassificationValues, + row['classification'], + 'classification', + ) as EmailClassification, classificationReason: row['classification_reason'] == null ? null @@ -171,13 +169,11 @@ export function mapMailboxMessageRow( row['provider_message_id'] == null ? null : String(row['provider_message_id']), - deliveryStatus: - deliveryStatusRaw != null && - (emailDeliveryStatusValues as ReadonlyArray).includes( - deliveryStatusRaw, - ) - ? (deliveryStatusRaw as EmailDeliveryStatus) - : null, + deliveryStatus: requireOptionalPersistedEnum( + emailDeliveryStatusValues, + row['delivery_status'], + 'deliveryStatus', + ) as EmailDeliveryStatus | null, deliveryStatusAt: row['delivery_status_at'] == null ? null @@ -202,7 +198,11 @@ export function mapMailboxAttachmentRow( contentId: row['content_id'] == null ? null : String(row['content_id']), disposition: row['disposition'] == null ? null : String(row['disposition']), size: Number(row['size'] ?? 0) || 0, - storageKind: String(row['storage_kind']), + storageKind: requirePersistedEnum( + mailboxStorageKindValues, + row['storage_kind'], + 'storageKind', + ) as MailboxStorageKind, storageKey: row['storage_key'] == null ? null : String(row['storage_key']), createdAt: String(row['created_at']), } @@ -211,16 +211,15 @@ export function mapMailboxAttachmentRow( export function mapMailboxDeliveryEventRow( row: Record, ): MailboxDeliveryEventRecord { - const eventType = String(row['event_type'] ?? '') return { id: String(row['id']), messageId: row['message_id'] == null ? null : String(row['message_id']), inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), - eventType: (emailDeliveryEventTypeValues as ReadonlyArray).includes( - eventType, - ) - ? (eventType as EmailDeliveryEventType) - : 'received', + eventType: requirePersistedEnum( + emailDeliveryEventTypeValues, + row['event_type'], + 'eventType', + ) as EmailDeliveryEventType, provider: row['provider'] == null ? null : String(row['provider']), providerMessageId: row['provider_message_id'] == null @@ -232,7 +231,11 @@ export function mapMailboxDeliveryEventRow( : String(row['provider_event_id']), detailJson: String(row['detail_json'] ?? '{}'), needsEffectReconcile: Number(row['needs_effect_reconcile'] ?? 0) === 1, - state: mapInboundState(row['state']), + state: requireOptionalPersistedEnum( + mailboxInboundDeliveryStateValues, + row['state'], + 'state', + ) as MailboxInboundDeliveryState | null, fingerprint: row['fingerprint'] == null ? null : String(row['fingerprint']), storageLease: row['storage_lease'] == null ? null : String(row['storage_lease']), @@ -287,9 +290,11 @@ export function mapMailboxDeliveryEventRow( row['usage_effect_lease_at'] == null ? null : String(row['usage_effect_lease_at']), - subscriptionEffectState: mapSubscriptionEffectState( + subscriptionEffectState: requireOptionalPersistedEnum( + mailboxSubscriptionEffectStateValues, row['subscription_effect_state'], - ), + 'subscriptionEffectState', + ) as MailboxSubscriptionEffectState | null, subscriptionEffectLease: row['subscription_effect_lease'] == null ? null @@ -315,5 +320,6 @@ export function mapMailboxDeliveryEventRow( ? null : String(row['subscription_effect_last_error']), createdAt: String(row['created_at']), + updatedAt: String(row['updated_at']), } } diff --git a/packages/worker/src/email/mailbox-retention.ts b/packages/worker/src/email/mailbox-retention.ts index 2a86713797..cd1cc3b6a8 100644 --- a/packages/worker/src/email/mailbox-retention.ts +++ b/packages/worker/src/email/mailbox-retention.ts @@ -1,8 +1,10 @@ +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { mailboxBlobDeleteMaxKeys, mailboxDeliveryEventRetentionDays, mailboxMessageRetentionDays, mailboxRetentionBatchSize, + mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, } from './mailbox-types.ts' import { type MailboxStore } from './mailbox-store.ts' @@ -14,12 +16,54 @@ const deliveryEventRetentionMs = export type MailboxRetentionPassResult = { /** True when at least one message blob delete failed (row retained). */ hadBlobDeleteFailures: boolean + /** True when expired rows remain after a successful bounded pass. */ + expiredWorkRemaining: boolean +} + +export type MailboxRetentionRescheduleKind = + | 'idle' + | 'backoff' + | 'continue' + | 'next-due' + +export type MailboxRetentionReschedule = { + kind: MailboxRetentionRescheduleKind + atMs: number | null +} + +/** + * Deterministic next-alarm choice after a retention pass. + * - Blob delete failures → hourly backoff only + * - Successful pass with expired rows remaining → near-immediate continuation + * - Otherwise → future due-time (or idle) + */ +export function computeMailboxRetentionReschedule(input: { + nowMs: number + hadBlobDeleteFailures: boolean + expiredWorkRemaining: boolean + nextDueAtMs: number | null + continuationDelayMs?: number + backoffMs?: number +}): MailboxRetentionReschedule { + const backoffMs = input.backoffMs ?? mailboxRetentionRetryDelayMs + const continuationDelayMs = + input.continuationDelayMs ?? mailboxRetentionContinuationDelayMs + if (input.hadBlobDeleteFailures) { + return { kind: 'backoff', atMs: input.nowMs + backoffMs } + } + if (input.expiredWorkRemaining) { + return { kind: 'continue', atMs: input.nowMs + continuationDelayMs } + } + if (input.nextDueAtMs == null) { + return { kind: 'idle', atMs: null } + } + return { kind: 'next-due', atMs: input.nextDueAtMs } } /** * Soonest retention due-time from stored rows, or null when empty. * Overdue rows yield timestamps in the past — callers must apply retry backoff - * instead of scheduling at now+1s. + * or continuation instead of scheduling at now+1s blindly. */ export function nextMailboxRetentionDueAtMs( store: MailboxStore, @@ -42,7 +86,7 @@ export function nextMailboxRetentionDueAtMs( } /** - * Schedule time for the next retention alarm. + * Schedule time for the next retention alarm on a write path. * - Future due-time → schedule at due-time * - Overdue work → bounded hourly backoff (avoids 1s storms on persistent R2 failure) */ @@ -79,9 +123,39 @@ async function deleteBlobKeys( } } +/** + * Build owner-safe R2 keys for an expired message. Never deletes arbitrary + * stored key strings — only canonical inbound raw MIME keys and matching + * external attachment keys. + */ +export function canonicalRetentionBlobKeys(input: { + ownerId: string + messageId: string + direction: 'inbound' | 'outbound' + attachments: Array<{ id: string; storage_key: string | null }> +}): Array { + const keys: Array = [] + if (input.direction === 'inbound') { + keys.push(emailRawMimeKey(input.ownerId, input.messageId)) + } + for (const attachment of input.attachments) { + if (attachment.storage_key == null) continue + const expected = emailAttachmentBlobKey( + input.ownerId, + input.messageId, + attachment.id, + ) + if (attachment.storage_key === expected) { + keys.push(expected) + } + } + return keys +} + async function pruneExpiredMessages(input: { store: MailboxStore blobs: Pick + ownerId: string }): Promise { const cutoff = new Date(Date.now() - messageRetentionMs).toISOString() const rows = input.store.listExpiredMessagesForRetention({ @@ -90,26 +164,29 @@ async function pruneExpiredMessages(input: { }) if (rows.length === 0) return false - const attachmentRows = input.store.listAttachmentStorageKeysForMessages( + const attachmentRows = input.store.listAttachmentsForRetention( rows.map((row) => row.id), ) - const attachmentKeysByMessageId = new Map>() + const attachmentsByMessageId = new Map< + string, + Array<{ id: string; storage_key: string | null }> + >() for (const attachment of attachmentRows) { - const keys = attachmentKeysByMessageId.get(attachment.message_id) - if (keys) keys.push(attachment.storage_key) + const list = attachmentsByMessageId.get(attachment.message_id) + if (list) list.push(attachment) else { - attachmentKeysByMessageId.set(attachment.message_id, [ - attachment.storage_key, - ]) + attachmentsByMessageId.set(attachment.message_id, [attachment]) } } let hadBlobDeleteFailures = false for (const row of rows) { - const keys = [ - ...(row.raw_mime_key ? [row.raw_mime_key] : []), - ...(attachmentKeysByMessageId.get(row.id) ?? []), - ] + const keys = canonicalRetentionBlobKeys({ + ownerId: input.ownerId, + messageId: row.id, + direction: row.direction, + attachments: attachmentsByMessageId.get(row.id) ?? [], + }) const blobsDeleted = await deleteBlobKeys(input.blobs, keys) if (!blobsDeleted) { hadBlobDeleteFailures = true @@ -124,14 +201,30 @@ export async function enforceMailboxRetention(input: { store: MailboxStore blobs: Pick }): Promise { - const hadBlobDeleteFailures = await pruneExpiredMessages(input) + const messageCutoff = new Date(Date.now() - messageRetentionMs).toISOString() const eventCutoff = new Date( Date.now() - deliveryEventRetentionMs, ).toISOString() + + const ownerId = input.store.getOwnerId() + let hadBlobDeleteFailures = false + if (ownerId) { + hadBlobDeleteFailures = await pruneExpiredMessages({ + store: input.store, + blobs: input.blobs, + ownerId, + }) + } + input.store.pruneExpiredDeliveryEvents({ cutoff: eventCutoff, limit: mailboxRetentionBatchSize, }) input.store.pruneOrphanThreads(mailboxRetentionBatchSize) - return { hadBlobDeleteFailures } + + const expiredWorkRemaining = + input.store.hasExpiredMessages(messageCutoff) || + input.store.hasExpiredDeliveryEvents(eventCutoff) + + return { hadBlobDeleteFailures, expiredWorkRemaining } } diff --git a/packages/worker/src/email/mailbox-retention.workers.test.ts b/packages/worker/src/email/mailbox-retention.workers.test.ts new file mode 100644 index 0000000000..f0871a0a38 --- /dev/null +++ b/packages/worker/src/email/mailbox-retention.workers.test.ts @@ -0,0 +1,243 @@ +import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { + computeMailboxRetentionReschedule, + Mailbox, + mailboxDeliveryEventRetentionDays, + mailboxMessageRetentionDays, + mailboxRetentionContinuationDelayMs, + mailboxRetentionRetryDelayMs, +} from './mailbox-do.ts' +import { + baseAttachment, + baseMessage, + baseThread, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' + +test('computeMailboxRetentionReschedule prefers backoff, then continue, then due-time', () => { + const nowMs = 1_700_000_000_000 + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: true, + expiredWorkRemaining: true, + nextDueAtMs: nowMs + 10_000, + }), + ).toEqual({ + kind: 'backoff', + atMs: nowMs + mailboxRetentionRetryDelayMs, + }) + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: false, + expiredWorkRemaining: true, + nextDueAtMs: nowMs + 10_000, + }), + ).toEqual({ + kind: 'continue', + atMs: nowMs + mailboxRetentionContinuationDelayMs, + }) + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: false, + expiredWorkRemaining: false, + nextDueAtMs: nowMs + 60_000, + }), + ).toEqual({ kind: 'next-due', atMs: nowMs + 60_000 }) + expect( + computeMailboxRetentionReschedule({ + nowMs, + hadBlobDeleteFailures: false, + expiredWorkRemaining: false, + nextDueAtMs: null, + }), + ).toEqual({ kind: 'idle', atMs: null }) +}) + +test('Mailbox retention deletes canonical R2 keys before metadata and backs off on failure', async () => { + silenceIncidentalRuntimeWarnings() + consoleWarn.mockImplementation((...args: Array) => { + const message = String(args[0] ?? '') + if (message.includes('mailbox-retention-blob-delete-failed')) return + }) + + const userId = uniqueUserId('retention') + const mailbox = rpcFor(userId) + const stub = stubFor(userId) + + const oldMessageAt = new Date( + Date.now() - (mailboxMessageRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const oldEventAt = new Date( + Date.now() - (mailboxDeliveryEventRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const freshAt = new Date().toISOString() + + const keepMessage = baseMessage(userId, { + id: 'keep-msg', + subject: 'fresh', + createdAt: freshAt, + updatedAt: freshAt, + }) + const dropMessage = baseMessage(userId, { + id: 'drop-msg', + threadId: 'drop-thread', + subject: 'old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + const failMessage = baseMessage(userId, { + id: 'fail-msg', + subject: 'fail-old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + const missingKeyMessage = baseMessage(userId, { + id: 'missing-key-msg', + subject: 'missing-stored-key', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + const dropAttachment = baseAttachment(userId, dropMessage.id, { + id: 'drop-att', + createdAt: oldMessageAt, + }) + + await mailbox.mirrorMessage({ + ownerId: userId, + thread: baseThread({ + id: 'drop-thread', + lastMessageAt: oldMessageAt, + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }), + message: dropMessage, + attachments: [dropAttachment], + }) + await mailbox.mirrorMessage({ ownerId: userId, message: keepMessage }) + await mailbox.mirrorMessage({ ownerId: userId, message: failMessage }) + await mailbox.mirrorMessage({ ownerId: userId, message: missingKeyMessage }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'old-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: oldEventAt, + updatedAt: oldEventAt, + needsEffectReconcile: false, + }, + }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: { + id: 'fresh-event', + messageId: keepMessage.id, + eventType: 'received', + provider: 'kody', + createdAt: freshAt, + updatedAt: freshAt, + needsEffectReconcile: false, + }, + }) + + const dropRawKey = emailRawMimeKey(userId, dropMessage.id) + const dropAttKey = emailAttachmentBlobKey( + userId, + dropMessage.id, + dropAttachment.id, + ) + const failRawKey = emailRawMimeKey(userId, failMessage.id) + const keepRawKey = emailRawMimeKey(userId, keepMessage.id) + const missingRawKey = emailRawMimeKey(userId, missingKeyMessage.id) + + await env.EMAIL_BLOBS.put(dropRawKey, 'drop-raw') + await env.EMAIL_BLOBS.put(dropAttKey, 'drop-att') + await env.EMAIL_BLOBS.put(failRawKey, 'fail-raw') + await env.EMAIL_BLOBS.put(keepRawKey, 'keep-raw') + await env.EMAIL_BLOBS.put(missingRawKey, 'missing-raw') + + // Stored key missing must not prevent deleting the canonical inbound key. + await runInDurableObject(stub, async (_instance: Mailbox, state) => { + state.storage.sql.exec( + `UPDATE email_messages SET raw_mime_key = NULL WHERE id = ?`, + missingKeyMessage.id, + ) + }) + + const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) + env.EMAIL_BLOBS.delete = (async (keys: string | Array) => { + const list = Array.isArray(keys) ? keys : [keys] + if (list.includes(failRawKey)) { + throw new Error('simulated R2 delete failure') + } + return await originalDelete(keys) + }) as typeof env.EMAIL_BLOBS.delete + + try { + const alarmBefore = Date.now() + const alarmAfterFailure = await runInDurableObject( + stub, + async (instance: Mailbox, state) => { + expect(instance).toBeInstanceOf(Mailbox) + await state.storage.deleteAlarm() + await instance.alarm() + return await state.storage.getAlarm() + }, + ) + // Overdue retained work (failed R2 delete) must retry with hourly backoff, + // not every second. + expect(alarmAfterFailure).toBeTypeOf('number') + expect(alarmAfterFailure).toBeGreaterThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs - 5_000, + ) + expect(alarmAfterFailure).toBeLessThanOrEqual( + alarmBefore + mailboxRetentionRetryDelayMs + 5_000, + ) + + expect(await env.EMAIL_BLOBS.get(dropRawKey)).toBeNull() + expect(await env.EMAIL_BLOBS.get(dropAttKey)).toBeNull() + expect(await env.EMAIL_BLOBS.get(missingRawKey)).toBeNull() + expect(await env.EMAIL_BLOBS.get(failRawKey)).not.toBeNull() + expect(await env.EMAIL_BLOBS.get(keepRawKey)).not.toBeNull() + + expect(await mailbox.getMessage({ messageId: dropMessage.id })).toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: dropMessage.id }), + ).toHaveLength(0) + expect(await mailbox.getThread({ threadId: 'drop-thread' })).toBeNull() + expect( + await mailbox.getMessage({ messageId: missingKeyMessage.id }), + ).toBeNull() + expect( + await mailbox.getMessage({ messageId: failMessage.id }), + ).toMatchObject({ + id: failMessage.id, + }) + expect( + await mailbox.getMessage({ messageId: keepMessage.id }), + ).toMatchObject({ + id: keepMessage.id, + }) + const events = await mailbox.listDeliveryEvents({ limit: 10 }) + expect(events.map((event) => event.id)).toEqual(['fresh-event']) + } finally { + env.EMAIL_BLOBS.delete = originalDelete + } + + await runInDurableObject(stub, async (instance: Mailbox) => { + await instance.alarm() + }) + expect(await env.EMAIL_BLOBS.get(failRawKey)).toBeNull() + expect(await mailbox.getMessage({ messageId: failMessage.id })).toBeNull() +}) diff --git a/packages/worker/src/email/mailbox-schema.ts b/packages/worker/src/email/mailbox-schema.ts index ef8bcdda5d..77a8c8948e 100644 --- a/packages/worker/src/email/mailbox-schema.ts +++ b/packages/worker/src/email/mailbox-schema.ts @@ -29,48 +29,10 @@ export function setMailboxMeta( ) } -/** Warm v1 objects lack promoted inbound columns. */ -function ensureDeliveryEventPromotedColumns( - sql: SqlStorage, - installedVersion: number | null, -) { - if (installedVersion == null || installedVersion >= mailboxSchemaVersion) { - return - } - const columns: Array = [ - `state TEXT`, - `fingerprint TEXT`, - `storage_lease TEXT`, - `storage_lease_at TEXT`, - `cleanup_lease TEXT`, - `cleanup_lease_at TEXT`, - `cleanup_retry_at TEXT`, - `expected_attachment_count INTEGER`, - `finalization_token TEXT`, - `reconcile_after TEXT`, - `dedupe_expires_at TEXT`, - `usage_effect_suppressed_at TEXT`, - `usage_started_at TEXT`, - `usage_effect_retry_at TEXT`, - `usage_effect_lease TEXT`, - `usage_effect_lease_at TEXT`, - `subscription_effect_state TEXT`, - `subscription_effect_lease TEXT`, - `subscription_effect_lease_at TEXT`, - `subscription_effect_retry_at TEXT`, - `subscription_effect_attempt_count INTEGER`, - `subscription_effect_dead_letter_at TEXT`, - `subscription_effect_last_error TEXT`, - ] - for (const column of columns) { - try { - sql.exec(`ALTER TABLE email_delivery_events ADD COLUMN ${column}`) - } catch { - // Column already present on a partially migrated object. - } - } -} - +/** + * Schema v1 — coherent ship shape (this DO class has never been deployed). + * No warm ALTER migrations; bump `mailboxSchemaVersion` only with a real plan. + */ export function initializeMailboxSchema(storage: DurableObjectStorage) { const sql = storage.sql sql.exec(` @@ -82,6 +44,20 @@ export function initializeMailboxSchema(storage: DurableObjectStorage) { const installedVersion = getMailboxMeta(storage, mailboxMetaSchemaVersionKey) if (installedVersion === mailboxSchemaVersion) return + /** + * Cloudflare DO runtime does not expose a reverse lookup from the object + * id to the `idFromName` string, so the human owner id is not + * introspectable inside the DO. Ownership for blob-key validation and + * cross-user rejection is an explicit write-contract field, persisted + * once in this singleton row. Data rows still have no `user_id` column. + */ + sql.exec(` + CREATE TABLE IF NOT EXISTS mailbox_owner_identity ( + singleton INTEGER PRIMARY KEY NOT NULL CHECK (singleton = 1), + owner_id TEXT NOT NULL + ) + `) + sql.exec(` CREATE TABLE IF NOT EXISTS email_threads ( id TEXT PRIMARY KEY NOT NULL, @@ -276,10 +252,10 @@ export function initializeMailboxSchema(storage: DurableObjectStorage) { subscription_effect_attempt_count INTEGER, subscription_effect_dead_letter_at TEXT, subscription_effect_last_error TEXT, - created_at TEXT NOT NULL + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL ) `) - ensureDeliveryEventPromotedColumns(sql, installedVersion) sql.exec( `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_message_id ON email_delivery_events(message_id)`, diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index ce5c591766..8bb0307f6a 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -1,14 +1,11 @@ import { assertMailboxCanonicalIsoTimestamp, assertMailboxClassification, - assertMailboxDeliveryEventType, assertMailboxDeliveryStatus, assertMailboxDirection, - assertMailboxInboundDeliveryState, assertMailboxNonEmptyString, assertMailboxProcessingStatus, assertMailboxStorageKind, - assertMailboxSubscriptionEffectState, assertOptionalMailboxCanonicalIsoTimestamp, decodeMailboxListCursor, encodeMailboxListCursor, @@ -31,21 +28,35 @@ import { import { type EmailDeliveryEventType, type EmailDeliveryStatus, + type EmailDirection, } from './types.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { boundedEmailBody, escapeLikePattern, mapMailboxAttachmentRow, - mapMailboxDeliveryEventRow, mapMailboxMessageRow, mapMailboxThreadRow, } from './mailbox-mappers.ts' +import { + deliveryEventOwnsMessage, + hasExpiredMailboxDeliveryEvents, + listMailboxDeliveryEvents, + oldestMailboxDeliveryEventCreatedAt, + pruneExpiredMailboxDeliveryEvents, + writeMailboxDeliveryEventRow, +} from './mailbox-delivery-events.ts' import { exportMailboxFromStore, listMailboxBlobReferences, } from './mailbox-export.ts' import { initializeMailboxSchema } from './mailbox-schema.ts' +export type MailboxUpsertResult = { + created: boolean + accepted: boolean +} + /** * SQLite write/query helpers for one Mailbox DO. No alarm / R2 side effects. */ @@ -64,7 +75,97 @@ export class MailboxStore { initializeMailboxSchema(this.storage) } - upsertThreadRow(thread: MailboxThreadInput) { + getOwnerId(): string | null { + const row = this.sql + .exec<{ owner_id: string }>( + `SELECT owner_id FROM mailbox_owner_identity + WHERE singleton = 1 + LIMIT 1`, + ) + .toArray()[0] + return row?.owner_id ?? null + } + + /** + * Persist owner once; reject cross-user writes. DO name is not + * introspectable — see mailbox_owner_identity DDL comment. + */ + assertOwner(ownerId: string): string { + const id = assertMailboxNonEmptyString(ownerId, 'ownerId') + const existing = this.getOwnerId() + if (existing == null) { + this.sql.exec( + `INSERT INTO mailbox_owner_identity (singleton, owner_id) + VALUES (1, ?)`, + id, + ) + return id + } + if (existing !== id) { + throw new Error( + `Mailbox ownerId mismatch; this object is bound to a different owner.`, + ) + } + return id + } + + validateMessageBlobKeys(input: { + ownerId: string + message: MailboxMessageInput + attachments?: Array + }) { + const messageId = assertMailboxNonEmptyString( + input.message.id, + 'message.id', + ) + const direction = assertMailboxDirection(input.message.direction) + const rawMimeKey = input.message.rawMimeKey ?? null + if (direction === 'inbound') { + const expected = emailRawMimeKey(input.ownerId, messageId) + if (rawMimeKey !== expected) { + throw new Error( + `Mailbox inbound rawMimeKey must equal emailRawMimeKey(ownerId, messageId).`, + ) + } + } else if (rawMimeKey != null) { + const expected = emailRawMimeKey(input.ownerId, messageId) + if (rawMimeKey !== expected) { + throw new Error( + `Mailbox outbound rawMimeKey must be null or equal emailRawMimeKey(ownerId, messageId).`, + ) + } + } + + if (input.attachments === undefined) return + for (const attachment of input.attachments) { + const attachmentId = assertMailboxNonEmptyString( + attachment.id, + 'attachment.id', + ) + const storageKind = assertMailboxStorageKind( + String(attachment.storageKind), + ) + const storageKey = attachment.storageKey ?? null + if (storageKind === 'external') { + const expected = emailAttachmentBlobKey( + input.ownerId, + messageId, + attachmentId, + ) + if (storageKey !== expected) { + throw new Error( + `Mailbox external attachment storageKey must equal emailAttachmentBlobKey(ownerId, messageId, attachmentId).`, + ) + } + } else if (storageKey != null) { + throw new Error( + `Mailbox ${storageKind} attachment storageKey must be null.`, + ) + } + } + } + + upsertThreadRow(thread: MailboxThreadInput): MailboxUpsertResult { const id = assertMailboxNonEmptyString(thread.id, 'thread.id') const lastMessageAt = assertMailboxCanonicalIsoTimestamp( thread.lastMessageAt, @@ -75,9 +176,18 @@ export class MailboxStore { 'thread.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( - thread.updatedAt?.trim() || mailboxNowIso(), + thread.updatedAt, 'thread.updatedAt', ) + const existing = this.sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_threads WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + if (existing && updatedAt < existing.updated_at) { + return { created: false, accepted: false } + } this.sql.exec( `INSERT INTO email_threads ( id, inbox_id, subject_normalized, root_message_id_header, @@ -88,7 +198,8 @@ export class MailboxStore { subject_normalized = excluded.subject_normalized, root_message_id_header = excluded.root_message_id_header, last_message_at = excluded.last_message_at, - updated_at = excluded.updated_at`, + updated_at = excluded.updated_at + WHERE excluded.updated_at >= email_threads.updated_at`, id, thread.inboxId ?? null, thread.subjectNormalized ?? '', @@ -97,14 +208,15 @@ export class MailboxStore { createdAt, updatedAt, ) + return { created: existing == null, accepted: true } } /** - * Upsert message metadata. Delivery status is monotonic by - * `delivery_status_at` (`>=` / equal timestamps may update; older must not - * regress a newer status from dual-write replay). + * Upsert message metadata. Whole-row updates require equal/newer + * `updatedAt`. Delivery status remains monotonic by `delivery_status_at` + * within an accepted snapshot. */ - upsertMessageRow(message: MailboxMessageInput) { + upsertMessageRow(message: MailboxMessageInput): MailboxUpsertResult { const id = assertMailboxNonEmptyString(message.id, 'message.id') const direction = assertMailboxDirection(message.direction) const processingStatus = assertMailboxProcessingStatus( @@ -131,7 +243,7 @@ export class MailboxStore { 'message.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( - message.updatedAt?.trim() || mailboxNowIso(), + message.updatedAt, 'message.updatedAt', ) const receivedAt = assertOptionalMailboxCanonicalIsoTimestamp( @@ -142,6 +254,15 @@ export class MailboxStore { message.sentAt, 'message.sentAt', ) + const existing = this.sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + if (existing && updatedAt < existing.updated_at) { + return { created: false, accepted: false } + } this.sql.exec( `INSERT INTO email_messages ( id, direction, inbox_id, thread_id, sender_identity_id, @@ -202,7 +323,8 @@ export class MailboxStore { error = excluded.error, received_at = excluded.received_at, sent_at = excluded.sent_at, - updated_at = excluded.updated_at`, + updated_at = excluded.updated_at + WHERE excluded.updated_at >= email_messages.updated_at`, id, direction, message.inboxId ?? null, @@ -236,6 +358,7 @@ export class MailboxStore { createdAt, updatedAt, ) + return { created: existing == null, accepted: true } } replaceAttachmentsForMessage( @@ -283,188 +406,11 @@ export class MailboxStore { } } - findDeliveryEventByProviderEventId( - providerEventId: string, - ): MailboxDeliveryEventRecord | null { - const row = this.sql - .exec>( - `SELECT * FROM email_delivery_events - WHERE provider_event_id = ? - LIMIT 1`, - providerEventId, - ) - .toArray()[0] - return row ? mapMailboxDeliveryEventRow(row) : null - } - - writeDeliveryEventRow(event: MailboxDeliveryEventInput): boolean { - const id = assertMailboxNonEmptyString(event.id, 'event.id') - const eventType = assertMailboxDeliveryEventType(event.eventType) - const providerEventId = event.providerEventId ?? null - if (providerEventId) { - const existingByProvider = - this.findDeliveryEventByProviderEventId(providerEventId) - if (existingByProvider && existingByProvider.id !== id) { - return false - } - } - const existed = this.sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, - id, - ) - .toArray()[0] - const createdAt = assertMailboxCanonicalIsoTimestamp( - event.createdAt?.trim() || mailboxNowIso(), - 'event.createdAt', - ) - const detailJson = - typeof event.detailJson === 'string' && event.detailJson.length > 0 - ? event.detailJson - : '{}' - const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 - const state = - event.state == null - ? null - : assertMailboxInboundDeliveryState(event.state) - const subscriptionEffectState = - event.subscriptionEffectState == null - ? null - : assertMailboxSubscriptionEffectState(event.subscriptionEffectState) - - this.sql.exec( - `INSERT INTO email_delivery_events ( - id, message_id, inbox_id, event_type, provider, - provider_message_id, provider_event_id, detail_json, - needs_effect_reconcile, state, fingerprint, - storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, - cleanup_retry_at, expected_attachment_count, finalization_token, - reconcile_after, dedupe_expires_at, - usage_effect_recorded_at, usage_effect_suppressed_at, usage_started_at, - usage_month, usage_bytes, usage_duration_ms, - usage_effect_retry_at, usage_effect_lease, usage_effect_lease_at, - subscription_effect_state, subscription_effect_lease, - subscription_effect_lease_at, subscription_effect_retry_at, - subscription_effect_attempt_count, subscription_effect_dead_letter_at, - subscription_effect_last_error, created_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(id) DO UPDATE SET - message_id = excluded.message_id, - inbox_id = excluded.inbox_id, - event_type = excluded.event_type, - provider = excluded.provider, - provider_message_id = excluded.provider_message_id, - provider_event_id = excluded.provider_event_id, - detail_json = excluded.detail_json, - needs_effect_reconcile = excluded.needs_effect_reconcile, - state = excluded.state, - fingerprint = excluded.fingerprint, - storage_lease = excluded.storage_lease, - storage_lease_at = excluded.storage_lease_at, - cleanup_lease = excluded.cleanup_lease, - cleanup_lease_at = excluded.cleanup_lease_at, - cleanup_retry_at = excluded.cleanup_retry_at, - expected_attachment_count = excluded.expected_attachment_count, - finalization_token = excluded.finalization_token, - reconcile_after = excluded.reconcile_after, - dedupe_expires_at = excluded.dedupe_expires_at, - usage_effect_recorded_at = excluded.usage_effect_recorded_at, - usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, - usage_started_at = excluded.usage_started_at, - usage_month = excluded.usage_month, - usage_bytes = excluded.usage_bytes, - usage_duration_ms = excluded.usage_duration_ms, - usage_effect_retry_at = excluded.usage_effect_retry_at, - usage_effect_lease = excluded.usage_effect_lease, - usage_effect_lease_at = excluded.usage_effect_lease_at, - subscription_effect_state = excluded.subscription_effect_state, - subscription_effect_lease = excluded.subscription_effect_lease, - subscription_effect_lease_at = excluded.subscription_effect_lease_at, - subscription_effect_retry_at = excluded.subscription_effect_retry_at, - subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, - subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, - subscription_effect_last_error = excluded.subscription_effect_last_error`, - id, - event.messageId ?? null, - event.inboxId ?? null, - eventType, - event.provider ?? 'kody', - event.providerMessageId ?? null, - providerEventId, - detailJson, - needsEffectReconcile, - state, - event.fingerprint ?? null, - event.storageLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.storageLeaseAt, - 'event.storageLeaseAt', - ), - event.cleanupLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.cleanupLeaseAt, - 'event.cleanupLeaseAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.cleanupRetryAt, - 'event.cleanupRetryAt', - ), - event.expectedAttachmentCount ?? null, - event.finalizationToken ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.reconcileAfter, - 'event.reconcileAfter', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.dedupeExpiresAt, - 'event.dedupeExpiresAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectRecordedAt, - 'event.usageEffectRecordedAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectSuppressedAt, - 'event.usageEffectSuppressedAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageStartedAt, - 'event.usageStartedAt', - ), - event.usageMonth ?? null, - event.usageBytes ?? null, - event.usageDurationMs ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectRetryAt, - 'event.usageEffectRetryAt', - ), - event.usageEffectLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.usageEffectLeaseAt, - 'event.usageEffectLeaseAt', - ), - subscriptionEffectState, - event.subscriptionEffectLease ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.subscriptionEffectLeaseAt, - 'event.subscriptionEffectLeaseAt', - ), - assertOptionalMailboxCanonicalIsoTimestamp( - event.subscriptionEffectRetryAt, - 'event.subscriptionEffectRetryAt', - ), - event.subscriptionEffectAttemptCount ?? null, - assertOptionalMailboxCanonicalIsoTimestamp( - event.subscriptionEffectDeadLetterAt, - 'event.subscriptionEffectDeadLetterAt', - ), - event.subscriptionEffectLastError ?? null, - createdAt, - ) - return existed == null + writeDeliveryEventRow(event: MailboxDeliveryEventInput): { + inserted: boolean + accepted: boolean + } { + return writeMailboxDeliveryEventRow(this.sql, event) } updateLatestDeliveryStatus(input: { @@ -495,17 +441,7 @@ export class MailboxStore { } deliveryEventOwnsMessage(eventId: string, messageId: string) { - return ( - this.sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_delivery_events - WHERE id = ? AND message_id = ? - LIMIT 1`, - eventId, - messageId, - ) - .toArray()[0] != null - ) + return deliveryEventOwnsMessage(this.sql, eventId, messageId) } getThread(threadId: string): MailboxThreadRecord | null { @@ -685,28 +621,7 @@ export class MailboxStore { eventType?: EmailDeliveryEventType | null limit?: number }): Array { - const limit = normalizeMailboxPageSize(input.limit) - const clauses: Array = ['1 = 1'] - const params: Array = [] - if (input.messageId) { - clauses.push('message_id = ?') - params.push(input.messageId) - } - if (input.eventType) { - clauses.push('event_type = ?') - params.push(assertMailboxDeliveryEventType(input.eventType)) - } - params.push(limit) - return this.sql - .exec>( - `SELECT * FROM email_delivery_events - WHERE ${clauses.join(' AND ')} - ORDER BY created_at DESC, id DESC - LIMIT ?`, - ...params, - ) - .toArray() - .map(mapMailboxDeliveryEventRow) + return listMailboxDeliveryEvents(this.sql, input) } countMailbox(): MailboxCountResult { @@ -741,7 +656,11 @@ export class MailboxStore { pageSize?: number startAfter?: string | null }): MailboxBlobReferencePage { - return listMailboxBlobReferences(this.sql, input) + return listMailboxBlobReferences(this.sql, { + ownerId: this.getOwnerId(), + pageSize: input.pageSize, + startAfter: input.startAfter, + }) } oldestMessageCreatedAt(): string | null { @@ -756,27 +675,24 @@ export class MailboxStore { } oldestDeliveryEventCreatedAt(): string | null { - const row = this.sql - .exec<{ created_at: string }>( - `SELECT created_at FROM email_delivery_events - ORDER BY created_at ASC, id ASC - LIMIT 1`, - ) - .toArray()[0] - return row?.created_at ?? null + return oldestMailboxDeliveryEventCreatedAt(this.sql) } listExpiredMessagesForRetention(input: { cutoff: string limit: number - }): Array<{ id: string; raw_mime_key: string | null; created_at: string }> { + }): Array<{ + id: string + direction: EmailDirection + created_at: string + }> { return this.sql .exec<{ id: string - raw_mime_key: string | null + direction: EmailDirection created_at: string }>( - `SELECT id, raw_mime_key, created_at FROM email_messages + `SELECT id, direction, created_at FROM email_messages WHERE created_at < ? ORDER BY created_at ASC, id ASC LIMIT ?`, @@ -786,15 +702,18 @@ export class MailboxStore { .toArray() } - listAttachmentStorageKeysForMessages( + listAttachmentsForRetention( messageIds: Array, - ): Array<{ message_id: string; storage_key: string }> { + ): Array<{ id: string; message_id: string; storage_key: string | null }> { if (messageIds.length === 0) return [] return this.sql - .exec<{ message_id: string; storage_key: string }>( - `SELECT message_id, storage_key FROM email_attachments - WHERE storage_key IS NOT NULL - AND message_id IN (${messageIds.map(() => '?').join(', ')})`, + .exec<{ + id: string + message_id: string + storage_key: string | null + }>( + `SELECT id, message_id, storage_key FROM email_attachments + WHERE message_id IN (${messageIds.map(() => '?').join(', ')})`, ...messageIds, ) .toArray() @@ -809,19 +728,26 @@ export class MailboxStore { } pruneExpiredDeliveryEvents(input: { cutoff: string; limit: number }) { - this.sql.exec( - `DELETE FROM email_delivery_events - WHERE id IN ( - SELECT id FROM email_delivery_events - WHERE created_at < ? - ORDER BY created_at ASC, id ASC - LIMIT ? - )`, - input.cutoff, - input.limit, + pruneExpiredMailboxDeliveryEvents(this.sql, input) + } + + hasExpiredMessages(cutoff: string): boolean { + return ( + this.sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_messages + WHERE created_at < ? + LIMIT 1`, + cutoff, + ) + .toArray()[0] != null ) } + hasExpiredDeliveryEvents(cutoff: string): boolean { + return hasExpiredMailboxDeliveryEvents(this.sql, cutoff) + } + pruneOrphanThreads(limit: number) { this.sql.exec( `DELETE FROM email_threads diff --git a/packages/worker/src/email/mailbox-test-helpers.ts b/packages/worker/src/email/mailbox-test-helpers.ts new file mode 100644 index 0000000000..3578f312b4 --- /dev/null +++ b/packages/worker/src/email/mailbox-test-helpers.ts @@ -0,0 +1,122 @@ +import { env } from 'cloudflare:workers' +import { expect } from 'vitest' +import { mailboxDurableObjectName } from '#worker/user-scoped-durable-object-name.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { + type MailboxAttachmentInput, + type MailboxMessageInput, + type MailboxThreadInput, +} from './mailbox-do.ts' + +export function mailboxEnv(): MailboxEnv & { MAILBOX: DurableObjectNamespace } { + const mailbox = (env as MailboxEnv).MAILBOX + if (!mailbox) { + throw new Error( + 'MAILBOX Durable Object binding is required for mailbox-do workers tests.', + ) + } + return { MAILBOX: mailbox } +} + +export function stubFor(userId: string) { + const { MAILBOX } = mailboxEnv() + return MAILBOX.get(MAILBOX.idFromName(mailboxDurableObjectName(userId))) +} + +export function rpcFor(userId: string) { + return mailboxRpc({ env: mailboxEnv(), userId }) +} + +export function uniqueUserId(label: string) { + return `mailbox-${label}-${crypto.randomUUID()}` +} + +export async function assertMailboxThrows( + pattern: RegExp, + run: () => Promise, +) { + try { + await run() + throw new Error(`Expected mailbox RPC to throw matching ${pattern}`) + } catch (error) { + expect(String(error)).toMatch(pattern) + } +} + +export function baseThread( + overrides?: Partial, +): MailboxThreadInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' + return { + id, + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: ``, + lastMessageAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + } +} + +export function baseMessage( + ownerId: string, + overrides?: Partial, +): MailboxMessageInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.createdAt ?? '2026-07-01T12:00:00.000Z' + const direction = overrides?.direction ?? 'inbound' + const rawMimeKey = + overrides?.rawMimeKey !== undefined + ? overrides.rawMimeKey + : direction === 'outbound' + ? null + : emailRawMimeKey(ownerId, id) + return { + inboxId: 'inbox-1', + threadId: null, + fromAddress: 'sender@example.com', + envelopeFrom: 'envelope@example.com', + toAddresses: ['owner@example.com'], + subject: 'Hello mailbox', + messageIdHeader: ``, + rawSize: 128, + processingStatus: 'stored', + classification: 'accepted', + receivedAt: at, + createdAt: at, + updatedAt: at, + ...overrides, + id, + direction, + rawMimeKey, + } +} + +export function baseAttachment( + ownerId: string, + messageId: string, + overrides?: Partial, +): MailboxAttachmentInput { + const id = overrides?.id ?? crypto.randomUUID() + const storageKind = overrides?.storageKind ?? 'external' + const storageKey = + overrides?.storageKey !== undefined + ? overrides.storageKey + : storageKind === 'external' + ? emailAttachmentBlobKey(ownerId, messageId, id) + : null + return { + filename: 'note.txt', + contentType: 'text/plain', + size: 12, + createdAt: '2026-07-01T12:00:00.000Z', + ...overrides, + id, + messageId, + storageKind, + storageKey, + } +} diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index a93016b2ab..1702e2b1b1 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -24,10 +24,12 @@ export const mailboxRetentionBatchSize = 100 export const mailboxRetentionAlarmSkewMs = 60_000 /** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ export const mailboxRetentionRetryDelayMs = 60 * 60 * 1000 +/** Near-immediate continuation when a retention pass succeeds but expired rows remain. */ +export const mailboxRetentionContinuationDelayMs = 1_000 export const mailboxBlobDeleteMaxKeys = 1000 /** Bump when initializeSchema's DDL set changes; warm objects skip DDL. */ -export const mailboxSchemaVersion = 2 +export const mailboxSchemaVersion = 1 export const mailboxMetaSchemaVersionKey = 'schema_version' export const mailboxExportThreadCursorPrefix = 'thread:' @@ -122,7 +124,7 @@ export type MailboxAttachmentRecord = { contentId: string | null disposition: string | null size: number - storageKind: string + storageKind: MailboxStorageKind storageKey: string | null createdAt: string } @@ -169,6 +171,7 @@ export type MailboxDeliveryEventRecord = { subscriptionEffectDeadLetterAt: string | null subscriptionEffectLastError: string | null createdAt: string + updatedAt: string } export type MailboxThreadInput = { @@ -178,7 +181,8 @@ export type MailboxThreadInput = { rootMessageIdHeader?: string | null lastMessageAt: string createdAt?: string - updatedAt?: string + /** Required for stale-snapshot rejection (equal/newer only). */ + updatedAt: string } export type MailboxMessageInput = { @@ -213,7 +217,8 @@ export type MailboxMessageInput = { receivedAt?: string | null sentAt?: string | null createdAt?: string - updatedAt?: string + /** Required for stale-snapshot rejection (equal/newer only). */ + updatedAt: string } export type MailboxAttachmentInput = { @@ -268,6 +273,8 @@ export type MailboxDeliveryEventInput = { subscriptionEffectDeadLetterAt?: string | null subscriptionEffectLastError?: string | null createdAt?: string + /** Required monotonic mirror timestamp (equal/newer accepted). */ + updatedAt: string } export type MailboxExportRow = @@ -334,18 +341,33 @@ export type MailboxExportPhase = export type MailboxRpc = { mirrorMessage: (input: { + /** + * Explicit owner binding. DO name is not introspectable at runtime, so + * writers must pass the owning user id for blob-key validation and + * singleton identity checks. + */ + ownerId: string thread?: MailboxThreadInput | null message: MailboxMessageInput + /** + * Omitted → leave existing attachment metadata unchanged. + * Explicit `[]` → clear attachments (only when the message snapshot is accepted). + */ attachments?: Array - }) => Promise<{ ok: true }> + }) => Promise<{ ok: true; accepted: boolean }> upsertDeliveryEvent: (input: { + ownerId: string event: MailboxDeliveryEventInput latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus deliveryStatusAt: string } | null - }) => Promise<{ inserted: boolean; updatedLatestStatus: boolean }> + }) => Promise<{ + inserted: boolean + accepted: boolean + updatedLatestStatus: boolean + }> getThread: (input: { threadId: string }) => Promise diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 6cedacf759..3503580936 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -50,30 +50,9 @@ export type EmailInboundDeliveryFence = { storageLease: string } -/** - * R2 object key for a message's raw MIME payload in the EMAIL_BLOBS - * bucket. The userId prefix is part of the per-user isolation contract: - * account deletion enumerates and deletes a user's blobs by these stored - * keys, and the key can never be forged to point at another user's mail. - * Writers always store this canonical key in `raw_mime_key`. - */ -export function emailRawMimeKey(userId: string, messageId: string) { - return `email-raw:v1:${userId}/${messageId}` -} +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' -/** - * R2 object key for an attachment stored on its own (storage_kind - * 'external'), used by outbound mail whose bytes never exist as raw MIME. - * The userId prefix follows the same per-user isolation contract as - * emailRawMimeKey. - */ -export function emailAttachmentBlobKey( - userId: string, - messageId: string, - attachmentId: string, -) { - return `email-attachment:v1:${userId}/${messageId}/${attachmentId}` -} +export { emailAttachmentBlobKey, emailRawMimeKey } // One corrupt stored row must not fail an entire list/search response, so // the record parser degrades to an empty shape on malformed JSON. From 309fc692849d8821d168285071315339455b13c3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 07:15:25 +0000 Subject: [PATCH 03/62] fix(email): make Mailbox mirrors full snapshots Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 19 +- .../src/email/mailbox-delivery-events.ts | 52 +++--- packages/worker/src/email/mailbox-do.ts | 52 ++++-- .../src/email/mailbox-do.workers.test.ts | 40 ++-- .../worker/src/email/mailbox-retention.ts | 34 ++++ .../email/mailbox-retention.workers.test.ts | 46 ++++- packages/worker/src/email/mailbox-store.ts | 83 ++++----- .../worker/src/email/mailbox-test-helpers.ts | 92 +++++++++- packages/worker/src/email/mailbox-types.ts | 172 ++++++++++-------- 9 files changed, 388 insertions(+), 202 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 99179b9d8a..f270fb639b 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -609,12 +609,15 @@ owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` - latest per-message delivery status on `email_messages.delivery_status`, kept separate from send-request `processing_status` -**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` require -`ownerId` and apply equal-or-newer `updatedAt` snapshots (stale snapshots are -ignored, not applied). On `mirrorMessage`, omitting `attachments` preserves -existing attachment metadata; an explicit `attachments: []` clears them. -Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment -`storageKey` values against the canonical builders for that `ownerId`. +**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` take +complete snapshots — every persisted field is explicit (nullable fields use +explicit `null`). They are not patch APIs. Both require `ownerId` and apply +equal-or-newer `updatedAt` snapshots (stale snapshots are ignored, not applied). +The only omission exception is the `mirrorMessage` `attachments` bundle: +omitting it preserves existing attachment rows; an explicit `attachments: []` +clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and +external attachment `storageKey` values against the canonical builders for that +`ownerId`. **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -623,7 +626,9 @@ than trusting stored key strings), then apply strict blob-before-row ordering for `EMAIL_BLOBS` (failed blob deletes skip the row for retry). After a retention pass: successful work with expired rows remaining schedules a near-immediate continuation (`mailboxRetentionContinuationDelayMs`); R2 delete -failures use hourly backoff (`mailboxRetentionRetryDelayMs`). +failures use hourly backoff (`mailboxRetentionRetryDelayMs`). Write-path alarm +selection never postpones an earlier existing alarm under sustained writes +(near-equal times within skew keep the existing alarm). Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state diff --git a/packages/worker/src/email/mailbox-delivery-events.ts b/packages/worker/src/email/mailbox-delivery-events.ts index 8be19c23cc..88e71b364e 100644 --- a/packages/worker/src/email/mailbox-delivery-events.ts +++ b/packages/worker/src/email/mailbox-delivery-events.ts @@ -5,7 +5,6 @@ import { assertMailboxNonEmptyString, assertMailboxSubscriptionEffectState, assertOptionalMailboxCanonicalIsoTimestamp, - mailboxNowIso, normalizeMailboxPageSize, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, @@ -34,7 +33,8 @@ export function writeMailboxDeliveryEventRow( ): { inserted: boolean; accepted: boolean } { const id = assertMailboxNonEmptyString(event.id, 'event.id') const eventType = assertMailboxDeliveryEventType(event.eventType) - const providerEventId = event.providerEventId ?? null + const provider = assertMailboxNonEmptyString(event.provider, 'event.provider') + const providerEventId = event.providerEventId if (providerEventId) { const existingByProvider = findDeliveryEventByProviderEventId( sql, @@ -51,7 +51,7 @@ export function writeMailboxDeliveryEventRow( ) .toArray()[0] const createdAt = assertMailboxCanonicalIsoTimestamp( - event.createdAt?.trim() || mailboxNowIso(), + event.createdAt, 'event.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -61,11 +61,13 @@ export function writeMailboxDeliveryEventRow( if (existing && updatedAt < existing.updated_at) { return { inserted: false, accepted: false } } - const detailJson = - typeof event.detailJson === 'string' && event.detailJson.length > 0 - ? event.detailJson - : '{}' - const needsEffectReconcile = event.needsEffectReconcile === true ? 1 : 0 + if (typeof event.detailJson !== 'string') { + throw new Error('Mailbox event.detailJson must be a string.') + } + if (typeof event.needsEffectReconcile !== 'boolean') { + throw new Error('Mailbox event.needsEffectReconcile must be a boolean.') + } + const needsEffectReconcile = event.needsEffectReconcile ? 1 : 0 const state = event.state == null ? null : assertMailboxInboundDeliveryState(event.state) const subscriptionEffectState = @@ -131,22 +133,22 @@ export function writeMailboxDeliveryEventRow( updated_at = excluded.updated_at WHERE excluded.updated_at >= email_delivery_events.updated_at`, id, - event.messageId ?? null, - event.inboxId ?? null, + event.messageId, + event.inboxId, eventType, - event.provider ?? 'kody', - event.providerMessageId ?? null, + provider, + event.providerMessageId, providerEventId, - detailJson, + event.detailJson, needsEffectReconcile, state, - event.fingerprint ?? null, - event.storageLease ?? null, + event.fingerprint, + event.storageLease, assertOptionalMailboxCanonicalIsoTimestamp( event.storageLeaseAt, 'event.storageLeaseAt', ), - event.cleanupLease ?? null, + event.cleanupLease, assertOptionalMailboxCanonicalIsoTimestamp( event.cleanupLeaseAt, 'event.cleanupLeaseAt', @@ -155,8 +157,8 @@ export function writeMailboxDeliveryEventRow( event.cleanupRetryAt, 'event.cleanupRetryAt', ), - event.expectedAttachmentCount ?? null, - event.finalizationToken ?? null, + event.expectedAttachmentCount, + event.finalizationToken, assertOptionalMailboxCanonicalIsoTimestamp( event.reconcileAfter, 'event.reconcileAfter', @@ -177,20 +179,20 @@ export function writeMailboxDeliveryEventRow( event.usageStartedAt, 'event.usageStartedAt', ), - event.usageMonth ?? null, - event.usageBytes ?? null, - event.usageDurationMs ?? null, + event.usageMonth, + event.usageBytes, + event.usageDurationMs, assertOptionalMailboxCanonicalIsoTimestamp( event.usageEffectRetryAt, 'event.usageEffectRetryAt', ), - event.usageEffectLease ?? null, + event.usageEffectLease, assertOptionalMailboxCanonicalIsoTimestamp( event.usageEffectLeaseAt, 'event.usageEffectLeaseAt', ), subscriptionEffectState, - event.subscriptionEffectLease ?? null, + event.subscriptionEffectLease, assertOptionalMailboxCanonicalIsoTimestamp( event.subscriptionEffectLeaseAt, 'event.subscriptionEffectLeaseAt', @@ -199,12 +201,12 @@ export function writeMailboxDeliveryEventRow( event.subscriptionEffectRetryAt, 'event.subscriptionEffectRetryAt', ), - event.subscriptionEffectAttemptCount ?? null, + event.subscriptionEffectAttemptCount, assertOptionalMailboxCanonicalIsoTimestamp( event.subscriptionEffectDeadLetterAt, 'event.subscriptionEffectDeadLetterAt', ), - event.subscriptionEffectLastError ?? null, + event.subscriptionEffectLastError, createdAt, updatedAt, ) diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 146a9ac7bf..11cabd8d6f 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -10,11 +10,11 @@ import { enforceMailboxRetention, mailboxRetentionAlarmAtMs, nextMailboxRetentionDueAtMs, + selectMailboxRetentionWriteAlarm, } from './mailbox-retention.ts' import { MailboxStore } from './mailbox-store.ts' import { assertMailboxNonEmptyString, - mailboxRetentionAlarmSkewMs, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, @@ -71,6 +71,8 @@ class MailboxBase extends DurableObject implements MailboxRpc { * * - Schedule at most one alarm for the soonest retention due-time. * - Overdue work on the write path uses an hourly retry delay (not now+1s). + * - Write-path ensure never postpones an earlier existing alarm (skew-equal + * times keep the existing alarm to avoid churn). * - After an alarm: blob failures → hourly backoff; successful pass with * expired rows remaining → near-immediate continuation; else future due. * - `retentionAlarmArmed` skips getAlarm/setAlarm on hot writes once armed. @@ -83,24 +85,33 @@ class MailboxBase extends DurableObject implements MailboxRpc { if (this.retentionIdleConfirmed) return const dueAtMs = nextMailboxRetentionDueAtMs(this.store) - const alarmAt = mailboxRetentionAlarmAtMs({ dueAtMs }) - if (alarmAt == null) { - this.retentionIdleConfirmed = true - this.retentionAlarmArmed = false - return - } - - this.retentionIdleConfirmed = false - const existing = await this.ctx.storage.getAlarm() - if ( - existing != null && - Math.abs(existing - alarmAt) < mailboxRetentionAlarmSkewMs - ) { - this.retentionAlarmArmed = true - return + const proposedAtMs = mailboxRetentionAlarmAtMs({ dueAtMs }) + const existingAtMs = await this.ctx.storage.getAlarm() + const selection = selectMailboxRetentionWriteAlarm({ + proposedAtMs, + existingAtMs, + }) + switch (selection.action) { + case 'idle': + this.retentionIdleConfirmed = true + this.retentionAlarmArmed = false + return + case 'keep-existing': + this.retentionIdleConfirmed = false + this.retentionAlarmArmed = true + return + case 'set': + this.retentionIdleConfirmed = false + await this.ctx.storage.setAlarm(selection.atMs) + this.retentionAlarmArmed = true + return + default: { + const exhaustive: never = selection + throw new Error( + `Unhandled retention write-alarm selection: ${String(exhaustive)}`, + ) + } } - await this.ctx.storage.setAlarm(alarmAt) - this.retentionAlarmArmed = true } private markRetentionDirty() { @@ -319,4 +330,7 @@ export { type MailboxThreadInput, type MailboxThreadRecord, } from './mailbox-types.ts' -export { computeMailboxRetentionReschedule } from './mailbox-retention.ts' +export { + computeMailboxRetentionReschedule, + selectMailboxRetentionWriteAlarm, +} from './mailbox-retention.ts' diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index dfb70c369f..c619df9701 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -7,6 +7,7 @@ import { Mailbox } from './mailbox-do.ts' import { assertMailboxThrows, baseAttachment, + baseDeliveryEvent, baseMessage, baseThread, rpcFor, @@ -289,7 +290,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', const first = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1', messageId: outbound.id, eventType: 'delivered', @@ -301,7 +302,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', needsEffectReconcile: false, state: 'received', fingerprint: 'fp-1', - }, + }), latestDeliveryStatus: { messageId: outbound.id, deliveryStatus: 'delivered', @@ -316,7 +317,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', const staleEvent = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1', messageId: outbound.id, eventType: 'deferred', @@ -327,7 +328,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', needsEffectReconcile: true, state: null, fingerprint: null, - }, + }), }) expect(staleEvent).toEqual({ inserted: false, @@ -347,7 +348,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', const duplicate = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1-dup', messageId: outbound.id, eventType: 'delivered', @@ -355,7 +356,7 @@ test('Mailbox stale snapshots, attachment omit/clear, delivery-event updatedAt', providerEventId: 'provider-event-1', createdAt: '2026-07-02T11:00:00.000Z', updatedAt: '2026-07-02T11:00:00.000Z', - }, + }), }) expect(duplicate).toEqual({ inserted: false, @@ -451,7 +452,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const first = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1', messageId: message.id, eventType: 'delivered', @@ -461,7 +462,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso createdAt: '2026-07-02T10:00:00.000Z', updatedAt: '2026-07-02T10:00:00.000Z', needsEffectReconcile: false, - }, + }), latestDeliveryStatus: { messageId: message.id, deliveryStatus: 'delivered', @@ -476,7 +477,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const inbound = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'inbound-delivery-1', messageId: 'inbound-msg-1', inboxId: 'inbox-1', @@ -487,10 +488,6 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso needsEffectReconcile: true, state: 'received', fingerprint: 'fp-abc', - storageLease: null, - storageLeaseAt: null, - cleanupLease: null, - cleanupLeaseAt: null, expectedAttachmentCount: 2, finalizationToken: 'lease-token-1', usageStartedAt: '2026-07-02T10:59:00.000Z', @@ -500,7 +497,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso subscriptionEffectState: 'pending', subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', detailJson: JSON.stringify({ recipient: 'owner@example.com' }), - }, + }), }) expect(inbound.inserted).toBe(true) const inboundRow = ( @@ -527,17 +524,18 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso recipient: 'owner@example.com', }) - // Default needsEffectReconcile is false unless explicitly true. + // Explicit needsEffectReconcile: false is stored as false. await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-default-reconcile', messageId: message.id, eventType: 'sent', provider: 'kody', createdAt: '2026-07-02T11:30:00.000Z', updatedAt: '2026-07-02T11:30:00.000Z', - }, + needsEffectReconcile: false, + }), }) const defaultReconcile = ( await mailbox.listDeliveryEvents({ messageId: message.id, limit: 20 }) @@ -546,7 +544,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const duplicate = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-1-dup', messageId: message.id, eventType: 'delivered', @@ -555,7 +553,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerEventId: 'provider-event-1', createdAt: '2026-07-02T11:00:00.000Z', updatedAt: '2026-07-02T11:00:00.000Z', - }, + }), latestDeliveryStatus: { messageId: message.id, deliveryStatus: 'delivered', @@ -567,7 +565,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso const stale = await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'evt-2', messageId: message.id, eventType: 'deferred', @@ -576,7 +574,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso providerEventId: 'provider-event-2', createdAt: '2026-07-02T09:00:00.000Z', updatedAt: '2026-07-02T09:00:00.000Z', - }, + }), latestDeliveryStatus: { messageId: message.id, deliveryStatus: 'deferred', diff --git a/packages/worker/src/email/mailbox-retention.ts b/packages/worker/src/email/mailbox-retention.ts index cd1cc3b6a8..3ce4c5bafc 100644 --- a/packages/worker/src/email/mailbox-retention.ts +++ b/packages/worker/src/email/mailbox-retention.ts @@ -3,6 +3,7 @@ import { mailboxBlobDeleteMaxKeys, mailboxDeliveryEventRetentionDays, mailboxMessageRetentionDays, + mailboxRetentionAlarmSkewMs, mailboxRetentionBatchSize, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, @@ -102,6 +103,39 @@ export function mailboxRetentionAlarmAtMs(input: { return input.dueAtMs } +export type MailboxRetentionWriteAlarmSelection = + | { action: 'idle' } + | { action: 'keep-existing' } + | { action: 'set'; atMs: number } + +/** + * Choose whether a write-path ensure should set, keep, or clear interest in + * an alarm. Sustained writes must never postpone an earlier existing alarm; + * near-equal times within skew keep the existing alarm to avoid churn. + */ +export function selectMailboxRetentionWriteAlarm(input: { + proposedAtMs: number | null + existingAtMs: number | null + skewMs?: number +}): MailboxRetentionWriteAlarmSelection { + if (input.proposedAtMs == null) { + return { action: 'idle' } + } + const existingAtMs = input.existingAtMs + if (existingAtMs == null) { + return { action: 'set', atMs: input.proposedAtMs } + } + const skewMs = input.skewMs ?? mailboxRetentionAlarmSkewMs + if (Math.abs(existingAtMs - input.proposedAtMs) < skewMs) { + return { action: 'keep-existing' } + } + // Existing fires sooner — never postpone retention for a later proposal. + if (existingAtMs < input.proposedAtMs) { + return { action: 'keep-existing' } + } + return { action: 'set', atMs: input.proposedAtMs } +} + async function deleteBlobKeys( blobs: Pick, keys: Array, diff --git a/packages/worker/src/email/mailbox-retention.workers.test.ts b/packages/worker/src/email/mailbox-retention.workers.test.ts index f0871a0a38..d30cf646be 100644 --- a/packages/worker/src/email/mailbox-retention.workers.test.ts +++ b/packages/worker/src/email/mailbox-retention.workers.test.ts @@ -11,9 +11,12 @@ import { mailboxMessageRetentionDays, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, + selectMailboxRetentionWriteAlarm, } from './mailbox-do.ts' +import { mailboxRetentionAlarmSkewMs } from './mailbox-types.ts' import { baseAttachment, + baseDeliveryEvent, baseMessage, baseThread, rpcFor, @@ -63,6 +66,41 @@ test('computeMailboxRetentionReschedule prefers backoff, then continue, then due ).toEqual({ kind: 'idle', atMs: null }) }) +test('selectMailboxRetentionWriteAlarm never postpones an earlier existing alarm', () => { + const earlier = 1_700_000_100_000 + const later = earlier + 60_000 + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: later, + existingAtMs: earlier, + }), + ).toEqual({ action: 'keep-existing' }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: earlier, + existingAtMs: later, + }), + ).toEqual({ action: 'set', atMs: earlier }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: earlier + Math.floor(mailboxRetentionAlarmSkewMs / 2), + existingAtMs: earlier, + }), + ).toEqual({ action: 'keep-existing' }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: earlier, + existingAtMs: null, + }), + ).toEqual({ action: 'set', atMs: earlier }) + expect( + selectMailboxRetentionWriteAlarm({ + proposedAtMs: null, + existingAtMs: earlier, + }), + ).toEqual({ action: 'idle' }) +}) + test('Mailbox retention deletes canonical R2 keys before metadata and backs off on failure', async () => { silenceIncidentalRuntimeWarnings() consoleWarn.mockImplementation((...args: Array) => { @@ -128,7 +166,7 @@ test('Mailbox retention deletes canonical R2 keys before metadata and backs off await mailbox.mirrorMessage({ ownerId: userId, message: missingKeyMessage }) await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'old-event', messageId: keepMessage.id, eventType: 'received', @@ -136,11 +174,11 @@ test('Mailbox retention deletes canonical R2 keys before metadata and backs off createdAt: oldEventAt, updatedAt: oldEventAt, needsEffectReconcile: false, - }, + }), }) await mailbox.upsertDeliveryEvent({ ownerId: userId, - event: { + event: baseDeliveryEvent({ id: 'fresh-event', messageId: keepMessage.id, eventType: 'received', @@ -148,7 +186,7 @@ test('Mailbox retention deletes canonical R2 keys before metadata and backs off createdAt: freshAt, updatedAt: freshAt, needsEffectReconcile: false, - }, + }), }) const dropRawKey = emailRawMimeKey(userId, dropMessage.id) diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 8bb0307f6a..75b38f9927 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -119,7 +119,7 @@ export class MailboxStore { 'message.id', ) const direction = assertMailboxDirection(input.message.direction) - const rawMimeKey = input.message.rawMimeKey ?? null + const rawMimeKey = input.message.rawMimeKey if (direction === 'inbound') { const expected = emailRawMimeKey(input.ownerId, messageId) if (rawMimeKey !== expected) { @@ -142,10 +142,8 @@ export class MailboxStore { attachment.id, 'attachment.id', ) - const storageKind = assertMailboxStorageKind( - String(attachment.storageKind), - ) - const storageKey = attachment.storageKey ?? null + const storageKind = assertMailboxStorageKind(attachment.storageKind) + const storageKey = attachment.storageKey if (storageKind === 'external') { const expected = emailAttachmentBlobKey( input.ownerId, @@ -172,7 +170,7 @@ export class MailboxStore { 'thread.lastMessageAt', ) const createdAt = assertMailboxCanonicalIsoTimestamp( - thread.createdAt?.trim() || mailboxNowIso(), + thread.createdAt, 'thread.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -201,9 +199,9 @@ export class MailboxStore { updated_at = excluded.updated_at WHERE excluded.updated_at >= email_threads.updated_at`, id, - thread.inboxId ?? null, - thread.subjectNormalized ?? '', - thread.rootMessageIdHeader ?? null, + thread.inboxId, + thread.subjectNormalized, + thread.rootMessageIdHeader, lastMessageAt, createdAt, updatedAt, @@ -222,9 +220,7 @@ export class MailboxStore { const processingStatus = assertMailboxProcessingStatus( message.processingStatus, ) - const classification = assertMailboxClassification( - message.classification ?? 'accepted', - ) + const classification = assertMailboxClassification(message.classification) const deliveryStatus = message.deliveryStatus == null ? null @@ -239,7 +235,7 @@ export class MailboxStore { ) } const createdAt = assertMailboxCanonicalIsoTimestamp( - message.createdAt?.trim() || mailboxNowIso(), + message.createdAt, 'message.createdAt', ) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -327,32 +323,32 @@ export class MailboxStore { WHERE excluded.updated_at >= email_messages.updated_at`, id, direction, - message.inboxId ?? null, - message.threadId ?? null, - message.senderIdentityId ?? null, - message.fromAddress ?? '', - message.envelopeFrom ?? null, - JSON.stringify(message.toAddresses ?? []), - JSON.stringify(message.ccAddresses ?? []), - JSON.stringify(message.bccAddresses ?? []), - JSON.stringify(message.replyToAddresses ?? []), - message.subject ?? '', - message.messageIdHeader ?? null, - message.inReplyToHeader ?? null, - JSON.stringify(message.references ?? []), - message.headers ? JSON.stringify(message.headers) : '{}', - message.authResults ?? null, + message.inboxId, + message.threadId, + message.senderIdentityId, + message.fromAddress, + message.envelopeFrom, + JSON.stringify(message.toAddresses), + JSON.stringify(message.ccAddresses), + JSON.stringify(message.bccAddresses), + JSON.stringify(message.replyToAddresses), + message.subject, + message.messageIdHeader, + message.inReplyToHeader, + JSON.stringify(message.references), + JSON.stringify(message.headers), + message.authResults, boundedEmailBody(message.textBody), boundedEmailBody(message.htmlBody), - message.rawMimeKey ?? null, - message.rawSize ?? 0, + message.rawMimeKey, + message.rawSize, processingStatus, classification, - message.classificationReason ?? null, - message.providerMessageId ?? null, + message.classificationReason, + message.providerMessageId, deliveryStatus, deliveryStatusAt, - message.error ?? null, + message.error, receivedAt, sentAt, createdAt, @@ -380,13 +376,14 @@ export class MailboxStore { 'Mailbox attachment.messageId must match the mirrored message id.', ) } - const storageKind = assertMailboxStorageKind( - String(attachment.storageKind), - ) + const storageKind = assertMailboxStorageKind(attachment.storageKind) const createdAt = assertMailboxCanonicalIsoTimestamp( - attachment.createdAt?.trim() || mailboxNowIso(), + attachment.createdAt, 'attachment.createdAt', ) + if (!Number.isFinite(attachment.size)) { + throw new Error('Mailbox attachment.size must be a finite number.') + } this.sql.exec( `INSERT INTO email_attachments ( id, message_id, filename, content_type, content_id, disposition, @@ -394,13 +391,13 @@ export class MailboxStore { ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, id, attachmentMessageId, - attachment.filename ?? null, - attachment.contentType ?? 'application/octet-stream', - attachment.contentId ?? null, - attachment.disposition ?? null, - Math.max(0, Math.trunc(Number(attachment.size) || 0)), + attachment.filename, + attachment.contentType, + attachment.contentId, + attachment.disposition, + Math.trunc(attachment.size), storageKind, - attachment.storageKey ?? null, + attachment.storageKey, createdAt, ) } diff --git a/packages/worker/src/email/mailbox-test-helpers.ts b/packages/worker/src/email/mailbox-test-helpers.ts index 3578f312b4..b647ea9d6a 100644 --- a/packages/worker/src/email/mailbox-test-helpers.ts +++ b/packages/worker/src/email/mailbox-test-helpers.ts @@ -5,6 +5,7 @@ import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' import { type MailboxAttachmentInput, + type MailboxDeliveryEventInput, type MailboxMessageInput, type MailboxThreadInput, } from './mailbox-do.ts' @@ -44,12 +45,13 @@ export async function assertMailboxThrows( } } +/** Complete thread snapshot; overrides may be Partial. */ export function baseThread( overrides?: Partial, ): MailboxThreadInput { const id = overrides?.id ?? crypto.randomUUID() const at = overrides?.lastMessageAt ?? '2026-07-01T12:00:00.000Z' - return { + const defaults: MailboxThreadInput = { id, inboxId: 'inbox-1', subjectNormalized: 'hello', @@ -57,10 +59,11 @@ export function baseThread( lastMessageAt: at, createdAt: at, updatedAt: at, - ...overrides, } + return { ...defaults, ...overrides, id: overrides?.id ?? id } } +/** Complete message snapshot; overrides may be Partial. */ export function baseMessage( ownerId: string, overrides?: Partial, @@ -74,20 +77,42 @@ export function baseMessage( : direction === 'outbound' ? null : emailRawMimeKey(ownerId, id) - return { + const defaults: MailboxMessageInput = { + id, + direction, inboxId: 'inbox-1', threadId: null, + senderIdentityId: null, fromAddress: 'sender@example.com', envelopeFrom: 'envelope@example.com', toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], subject: 'Hello mailbox', messageIdHeader: ``, + inReplyToHeader: null, + references: [], + headers: {}, + authResults: null, + textBody: null, + htmlBody: null, + rawMimeKey, rawSize: 128, processingStatus: 'stored', classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, receivedAt: at, + sentAt: null, createdAt: at, updatedAt: at, + } + return { + ...defaults, ...overrides, id, direction, @@ -95,6 +120,7 @@ export function baseMessage( } } +/** Complete attachment snapshot; overrides may be Partial. */ export function baseAttachment( ownerId: string, messageId: string, @@ -108,11 +134,20 @@ export function baseAttachment( : storageKind === 'external' ? emailAttachmentBlobKey(ownerId, messageId, id) : null - return { + const defaults: MailboxAttachmentInput = { + id, + messageId, filename: 'note.txt', contentType: 'text/plain', + contentId: null, + disposition: null, size: 12, + storageKind, + storageKey, createdAt: '2026-07-01T12:00:00.000Z', + } + return { + ...defaults, ...overrides, id, messageId, @@ -120,3 +155,52 @@ export function baseAttachment( storageKey, } } + +/** Complete delivery-event snapshot; overrides may be Partial. */ +export function baseDeliveryEvent( + overrides?: Partial, +): MailboxDeliveryEventInput { + const id = overrides?.id ?? crypto.randomUUID() + const at = overrides?.createdAt ?? '2026-07-02T10:00:00.000Z' + const defaults: MailboxDeliveryEventInput = { + id, + messageId: null, + inboxId: null, + eventType: 'received', + provider: 'kody', + providerMessageId: null, + providerEventId: null, + detailJson: '{}', + needsEffectReconcile: false, + state: null, + fingerprint: null, + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: null, + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: null, + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: at, + updatedAt: at, + } + return { ...defaults, ...overrides, id: overrides?.id ?? id } +} diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 1702e2b1b1..6e281c850c 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -174,106 +174,120 @@ export type MailboxDeliveryEventRecord = { updatedAt: string } +/** + * Full thread snapshot for mirror writes — every persisted field is required. + * Nullable columns use `T | null` (no omitted/defaulted fields). + */ export type MailboxThreadInput = { id: string - inboxId?: string | null - subjectNormalized?: string | null - rootMessageIdHeader?: string | null + inboxId: string | null + subjectNormalized: string + rootMessageIdHeader: string | null lastMessageAt: string - createdAt?: string - /** Required for stale-snapshot rejection (equal/newer only). */ + createdAt: string + /** Stale-snapshot rejection: equal/newer only. */ updatedAt: string } +/** + * Full message snapshot for mirror writes — every persisted field is required. + * Nullable columns use `T | null` (no omitted/defaulted fields). + */ export type MailboxMessageInput = { id: string direction: EmailDirection - inboxId?: string | null - threadId?: string | null - senderIdentityId?: string | null - fromAddress?: string | null - envelopeFrom?: string | null - toAddresses?: Array - ccAddresses?: Array - bccAddresses?: Array - replyToAddresses?: Array - subject?: string | null - messageIdHeader?: string | null - inReplyToHeader?: string | null - references?: Array - headers?: Record | null - authResults?: string | null - textBody?: string | null - htmlBody?: string | null - rawMimeKey?: string | null - rawSize?: number | null + inboxId: string | null + threadId: string | null + senderIdentityId: string | null + fromAddress: string + envelopeFrom: string | null + toAddresses: Array + ccAddresses: Array + bccAddresses: Array + replyToAddresses: Array + subject: string + messageIdHeader: string | null + inReplyToHeader: string | null + references: Array + headers: Record + authResults: string | null + textBody: string | null + htmlBody: string | null + rawMimeKey: string | null + rawSize: number processingStatus: EmailProcessingStatus - classification?: EmailClassification - classificationReason?: string | null - providerMessageId?: string | null - deliveryStatus?: EmailDeliveryStatus | null - deliveryStatusAt?: string | null - error?: string | null - receivedAt?: string | null - sentAt?: string | null - createdAt?: string - /** Required for stale-snapshot rejection (equal/newer only). */ + classification: EmailClassification + classificationReason: string | null + providerMessageId: string | null + deliveryStatus: EmailDeliveryStatus | null + deliveryStatusAt: string | null + error: string | null + receivedAt: string | null + sentAt: string | null + createdAt: string + /** Stale-snapshot rejection: equal/newer only. */ updatedAt: string } +/** + * Full attachment snapshot — every persisted field is required. + * The RPC `attachments` array itself may still be omitted (preserve) or `[]` (clear). + */ export type MailboxAttachmentInput = { id: string messageId: string - filename?: string | null - contentType?: string | null - contentId?: string | null - disposition?: string | null - size?: number - storageKind: MailboxStorageKind | string - storageKey?: string | null - createdAt?: string + filename: string | null + contentType: string + contentId: string | null + disposition: string | null + size: number + storageKind: MailboxStorageKind + storageKey: string | null + createdAt: string } +/** + * Full delivery-event snapshot — every persisted field is required. + * `updatedAt` is the monotonic mirror timestamp (equal/newer accepted). + */ export type MailboxDeliveryEventInput = { id: string - messageId?: string | null - inboxId?: string | null + messageId: string | null + inboxId: string | null eventType: EmailDeliveryEventType - provider?: string | null - providerMessageId?: string | null - providerEventId?: string | null - detailJson?: string - /** Defaults false — set true only when inbound effects still need reconcile. */ - needsEffectReconcile?: boolean - state?: MailboxInboundDeliveryState | null - fingerprint?: string | null - storageLease?: string | null - storageLeaseAt?: string | null - cleanupLease?: string | null - cleanupLeaseAt?: string | null - cleanupRetryAt?: string | null - expectedAttachmentCount?: number | null - finalizationToken?: string | null - reconcileAfter?: string | null - dedupeExpiresAt?: string | null - usageEffectRecordedAt?: string | null - usageEffectSuppressedAt?: string | null - usageStartedAt?: string | null - usageMonth?: string | null - usageBytes?: number | null - usageDurationMs?: number | null - usageEffectRetryAt?: string | null - usageEffectLease?: string | null - usageEffectLeaseAt?: string | null - subscriptionEffectState?: MailboxSubscriptionEffectState | null - subscriptionEffectLease?: string | null - subscriptionEffectLeaseAt?: string | null - subscriptionEffectRetryAt?: string | null - subscriptionEffectAttemptCount?: number | null - subscriptionEffectDeadLetterAt?: string | null - subscriptionEffectLastError?: string | null - createdAt?: string - /** Required monotonic mirror timestamp (equal/newer accepted). */ + provider: string + providerMessageId: string | null + providerEventId: string | null + detailJson: string + needsEffectReconcile: boolean + state: MailboxInboundDeliveryState | null + fingerprint: string | null + storageLease: string | null + storageLeaseAt: string | null + cleanupLease: string | null + cleanupLeaseAt: string | null + cleanupRetryAt: string | null + expectedAttachmentCount: number | null + finalizationToken: string | null + reconcileAfter: string | null + dedupeExpiresAt: string | null + usageEffectRecordedAt: string | null + usageEffectSuppressedAt: string | null + usageStartedAt: string | null + usageMonth: string | null + usageBytes: number | null + usageDurationMs: number | null + usageEffectRetryAt: string | null + usageEffectLease: string | null + usageEffectLeaseAt: string | null + subscriptionEffectState: MailboxSubscriptionEffectState | null + subscriptionEffectLease: string | null + subscriptionEffectLeaseAt: string | null + subscriptionEffectRetryAt: string | null + subscriptionEffectAttemptCount: number | null + subscriptionEffectDeadLetterAt: string | null + subscriptionEffectLastError: string | null + createdAt: string updatedAt: string } From cebf5b37512606d1d5a0d0aad605b14de0726f67 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 09:12:54 +0000 Subject: [PATCH 04/62] feat(email): add Mailbox dual-write primitives Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 116 +++++- packages/worker/src/email/email-owner.ts | 14 + packages/worker/src/email/mailbox-do.ts | 129 +++++++ .../src/email/mailbox-mirror.node.test.ts | 333 ++++++++++++++++ packages/worker/src/email/mailbox-mirror.ts | 261 +++++++++++++ .../worker/src/email/mailbox-mutations.ts | 228 +++++++++++ .../email/mailbox-mutations.workers.test.ts | 357 +++++++++++++++++ .../email/mailbox-parity-events.node.test.ts | 152 ++++++++ .../worker/src/email/mailbox-parity-events.ts | 87 +++++ .../src/email/mailbox-snapshots.node.test.ts | 309 +++++++++++++++ .../worker/src/email/mailbox-snapshots.ts | 362 ++++++++++++++++++ packages/worker/src/email/mailbox-store.ts | 45 +++ packages/worker/src/email/mailbox-types.ts | 93 +++++ packages/worker/src/email/system-email.ts | 3 +- 14 files changed, 2472 insertions(+), 17 deletions(-) create mode 100644 packages/worker/src/email/email-owner.ts create mode 100644 packages/worker/src/email/mailbox-mirror.node.test.ts create mode 100644 packages/worker/src/email/mailbox-mirror.ts create mode 100644 packages/worker/src/email/mailbox-mutations.ts create mode 100644 packages/worker/src/email/mailbox-mutations.workers.test.ts create mode 100644 packages/worker/src/email/mailbox-parity-events.node.test.ts create mode 100644 packages/worker/src/email/mailbox-parity-events.ts create mode 100644 packages/worker/src/email/mailbox-snapshots.node.test.ts create mode 100644 packages/worker/src/email/mailbox-snapshots.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 6bbeda7bb7..e7c38ddfcf 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -218,10 +218,10 @@ Durable Object export behavior: - `Mailbox` exports per-user email metadata (threads, messages, attachments, delivery events) through the account-export `mailbox` section (`exportMailbox` RPC; keyset pagination with prefixed cursors over those tables). Manifest - counts use `countMailbox`. Phase 1 registers this consumption without - dual-write or a change of D1 authority — D1 `email_*` rows remain the live - source of truth and are still exported in the `d1` section. See - [Mailbox](#durable-objects-mailbox). + counts use `countMailbox`. Phase 1 registers this consumption; phase 2a adds + dual-write primitives without live callers or a change of D1 authority — D1 + `email_*` rows remain the live source of truth and are still exported in the + `d1` section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -376,6 +376,28 @@ When Analytics Engine SQL is unreachable, these two charts zero-fill while the rest of the page renders. Local development uses the existing D1 counters and delivery-event table because Wrangler's emulated dataset has no SQL API. +**Mailbox expand-phase parity events** (additive; no live writers in the +phase-2a PR) reuse the same `EMAIL_EVENTS` dataset with a separate row shape +defined in `packages/worker/src/email/mailbox-parity-events.ts` +(`recordMailboxParityEvent`). These rows are for future dual-write observability +and reconciliation — they do not feed the admin insights charts today: + +- `index1` — stable user id (per-user isolation; `system:email` is excluded) +- `blob1` — mirror or parity operation name +- `blob2` — operation outcome +- `blob3` — source event timestamp (ISO 8601) +- `double1` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; + `0` when absent or not applicable) +- `double2` — event weight (always `1`) + +Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, +`update_message_delivery`, `set_message_classification`, +`delete_message_metadata`, `delete_delivery_event`. Mirror outcomes: `mirrored`, +`stale`, `skipped`, `error`. Parity operations: `compare_threads`, +`compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity +outcomes: `match`, `mismatch`. Writes are best-effort and never throw into D1 +authority paths. + Two D1 reporting projections deliberately remain: - `usage_rollups` keeps 24 months of per-user monthly aggregates. Analytics @@ -640,6 +662,56 @@ clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment `storageKey` values against the canonical builders for that `ownerId`. +**Partial mutation RPCs** (phase 2a; owner-bound, monotonic on `updatedAt`, no +R2 or alarm side effects beyond the usual retention dirty mark): + +- `touchThread` — advance `last_message_at` / `updated_at` without a full + snapshot; `last_message_at` never moves backward +- `updateMessageDelivery` — outbound processing fields (`processing_status`, + `provider_message_id`, `error`, `sent_at`) +- `setMessageClassification` — inbound classification fields +- `deleteMessageMetadata` — metadata-only delete (attachments + message + orphan + thread); never deletes R2; distinguishes missing (idempotent) from stale + (newer `updated_at` retained) +- `deleteDeliveryEvent` — metadata-only delivery-event delete with the same + missing vs stale semantics + +Implementation lives in `mailbox-mutations.ts` (SQLite helpers) and is exposed +through `mailbox-do.ts` RPCs. All require `ownerId` and reject cross-owner +calls. + +**Email owner constant** (`packages/worker/src/email/email-owner.ts`): a +lightweight module exporting `systemEmailOwnerId` (`'system:email'`) and +`isSystemEmailOwner`. It stays free of system-email service imports so +dual-write helpers can skip the reserved operator inbox without import cycles. +Operator mail is never mirrored into per-user Mailbox objects. + +**D1 → Mailbox snapshot adapters** (`mailbox-snapshots.ts`): pure converters +that turn D1 rows into complete Mailbox wire inputs — `toMailboxThreadInput`, +`toMailboxMessageInput`, `toMailboxAttachmentInput`, and +`toMailboxDeliveryEventInput`. They normalize nulls to Mailbox SQLite defaults +(empty strings, `[]`, `{}`, `0`, `application/octet-stream`, `kody`) and fail +clearly on invalid persisted JSON/enums rather than fabricating state. Delivery +events take an additive `EmailDeliveryEventMirrorSnapshot`: D1-promoted columns +(`needs_effect_reconcile`, usage-effect fields) are authoritative; remaining +inbound/effect lease fields come from `detail_json`. D1 delivery events lack +`updated_at`; callers pass it explicitly for stale rejection. + +**Best-effort mirror helpers** (`mailbox-mirror.ts`): non-throwing wrappers +around the DO RPCs for future D1-authoritative dual-write. Each returns a +structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, +`{ status: 'stale' }`, `{ status: 'skipped', reason }` (`system-email` | +`mailbox-unconfigured` | `missing-owner`), or `{ status: 'error', error }`. +Failures log with stable tags (for example `mailbox-mirror-message-failed`) and +never propagate into D1 commit paths. Helpers cover full snapshots +(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot`) and +partial mutations (`mirrorMailboxTouchThread`, +`mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, +`mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`). +**There are no live dual-write callers in the phase-2a PR** — only unit/worker +tests invoke these helpers. D1 remains sole authority for all live mail +read/write paths. + **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). Alarm-driven deletes derive canonical blob keys from `ownerId` + row ids (rather @@ -660,13 +732,17 @@ mail content. Account export pages Mailbox state through the `mailbox` section ### Expand/contract phases -This is an expand/contract migration. **The current PR is phase 1 only:** -additive scaffold (DO class, binding, naming, client, mirror/export/purge RPCs, -and alarm retention) **plus registered account-deletion `purge` and -account-export `mailbox` section consumption**. It does **not** dual-write, does -**not** change live email read/write authority, and leaves D1 `email_*` tables -authoritative for all live paths. D1 email rows and existing R2 inventory -deletion stay authoritative during expand. +This is an expand/contract migration. **The current PR is phase 2a — +prerequisite scaffolding within phase 2:** additive dual-write **primitives** +(email-owner constant, D1→Mailbox snapshot adapters, best-effort mirror helpers +with structured outcomes, owner-bound partial-mutation and stale-safe +metadata-delete RPCs, and the Analytics Engine parity event schema). It does +**not** wire live dual-write callers, does **not** change live email read/write +authority, and leaves D1 `email_*` tables authoritative for all live paths. D1 +email rows and existing R2 inventory deletion stay authoritative during expand. +**Phase 2 is complete only when every user-mail mutation that writes D1 also +writes the DO and parity/reconciliation are wired** — this PR ships the library +surface that later live-path wiring will call. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -675,9 +751,16 @@ deletion stay authoritative during expand. authority for live mail. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO - completeness against D1. D1 stays authoritative for reads. -3. **Reads cut over after production parity** — user-mail reads move to the DO - only after production parity is verified. D1 writes continue. + completeness against D1. D1 stays authoritative for reads. Phase 2a (current + PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, `mailbox-mirror.ts`, + `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, + `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, + `deleteDeliveryEvent`), and `mailbox-parity-events.ts`. Those helpers and + parity-event writers are library-only today — **no production path invokes + them yet**; live-path dual-write and parity wiring remain for the rest of + phase 2. +3. **Reads cut over after production soak** — user-mail reads move to the DO + only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata to D1; retire dual-write and event/mirror machinery used only for the migration. @@ -723,7 +806,7 @@ metadata store: message (logged, non-retry) rather than risking a duplicate on Email Routing retry. -Phase 1 scaffolding does not change that live D1 boundary. +Phase 1 and phase 2a scaffolding does not change that live D1 boundary. ### Package state model @@ -844,7 +927,8 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` reconciliation alarms) - `MAILBOX` (Durable Objects; per-user email metadata — see [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export - consumption without changing D1 authority) + consumption; phase 2a adds dual-write primitives without changing D1 + authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) diff --git a/packages/worker/src/email/email-owner.ts b/packages/worker/src/email/email-owner.ts new file mode 100644 index 0000000000..697abd5a3c --- /dev/null +++ b/packages/worker/src/email/email-owner.ts @@ -0,0 +1,14 @@ +/** + * Lightweight email ownership constants. + * + * Kept free of system-email / service imports so dual-write helpers + * (`mailbox-mirror`) can skip the reserved operator inbox without creating + * import cycles (service → mailbox-mirror → … → service). + */ + +/** Reserved owner id for the platform operator inbox (stays in D1 only). */ +export const systemEmailOwnerId = 'system:email' + +export function isSystemEmailOwner(ownerId: string): boolean { + return ownerId === systemEmailOwnerId +} diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 11cabd8d6f..5222063275 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -15,10 +15,15 @@ import { import { MailboxStore } from './mailbox-store.ts' import { assertMailboxNonEmptyString, + type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataInput, + type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, @@ -27,8 +32,11 @@ import { type MailboxMessageRecord, type MailboxRpc, type MailboxSearchMessagesInput, + type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' /** @@ -216,6 +224,119 @@ class MailboxBase extends DurableObject implements MailboxRpc { return { inserted, accepted, updatedLatestStatus } } + /** + * Advance thread activity without a full snapshot. Equal/newer `updatedAt` + * only; `last_message_at` never moves backward. + */ + async touchThread( + input: MailboxTouchThreadInput, + ): Promise { + let accepted = false + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + accepted = this.store.touchThread({ + threadId: input.threadId, + lastMessageAt: input.lastMessageAt, + updatedAt: input.updatedAt, + }).accepted + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { accepted } + } + + /** + * Partial delivery/processing update for dual-write parity. Equal/newer + * `updatedAt` only. + */ + async updateMessageDelivery( + input: MailboxUpdateMessageDeliveryInput, + ): Promise { + let accepted = false + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + accepted = this.store.updateMessageDelivery({ + messageId: input.messageId, + processingStatus: input.processingStatus, + providerMessageId: input.providerMessageId, + error: input.error, + sentAt: input.sentAt, + updatedAt: input.updatedAt, + }).accepted + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { accepted } + } + + /** + * Partial classification update for dual-write parity. Equal/newer + * `updatedAt` only. + */ + async setMessageClassification( + input: MailboxSetMessageClassificationInput, + ): Promise { + let accepted = false + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + accepted = this.store.setMessageClassification({ + messageId: input.messageId, + classification: input.classification, + classificationReason: input.classificationReason, + updatedAt: input.updatedAt, + }).accepted + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { accepted } + } + + /** + * Metadata-only delete (attachments + message + orphan thread). Never + * deletes R2. Stale when the message `updated_at` is newer than `deletedAt`. + */ + async deleteMessageMetadata( + input: MailboxDeleteMessageMetadataInput, + ): Promise { + let result: MailboxDeleteMessageMetadataResult = { + deleted: false, + stale: false, + } + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = this.store.deleteMessageMetadata({ + messageId: input.messageId, + deletedAt: input.deletedAt, + }) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return result + } + + /** + * Metadata-only delivery-event delete. Distinguishes missing (idempotent) + * from stale (newer `updated_at` retained). + */ + async deleteDeliveryEvent( + input: MailboxDeleteDeliveryEventInput, + ): Promise { + let result: MailboxDeleteDeliveryEventResult = { + deleted: false, + stale: false, + } + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = this.store.deleteDeliveryEvent({ + eventId: input.eventId, + deletedAt: input.deletedAt, + }) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return result + } + async getThread(input: { threadId: string }): Promise { @@ -316,19 +437,27 @@ export { mailboxMessageRetentionDays, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, + type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReference, type MailboxBlobReferencePage, type MailboxCountResult, + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataInput, + type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, type MailboxExportRow, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' export { computeMailboxRetentionReschedule, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts new file mode 100644 index 0000000000..4668e5bb64 --- /dev/null +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -0,0 +1,333 @@ +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import { + mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxDeleteDeliveryEvent, + mirrorMailboxDeleteMessageMetadata, + mirrorMailboxMessageSnapshot, + mirrorMailboxSetMessageClassification, + mirrorMailboxTouchThread, + mirrorMailboxUpdateMessageDelivery, +} from './mailbox-mirror.ts' +import { + type EmailDeliveryEventRecord, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' + +function fakeMailboxEnv(stub: Record) { + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const get = vi.fn(() => stub) + return { + env: { + MAILBOX: { + idFromName, + get, + } as unknown as DurableObjectNamespace, + }, + idFromName, + get, + } +} + +function baseMessage( + overrides?: Partial, +): EmailMessageRecord { + return { + id: 'msg-1', + direction: 'inbound', + userId: 'user-aaa', + inboxId: 'inbox-1', + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: 'sender@example.com', + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: 'Hello', + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: {}, + authResults: null, + textBody: null, + htmlBody: null, + rawMimeKey: 'user-aaa/msg-1.eml', + rawSize: 32, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + ...overrides, + } +} + +test('mailbox mirror helpers scope by user idFromName, convert payloads, and report stale/skip/error', async () => { + consoleWarn.mockImplementation(() => {}) + + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const touchThread = vi.fn(async () => ({ accepted: true })) + const updateMessageDelivery = vi.fn(async () => ({ accepted: false })) + const setMessageClassification = vi.fn(async () => ({ accepted: true })) + const deleteMessageMetadata = vi.fn( + async (input: { messageId: string; deletedAt: string }) => { + if (input.messageId === 'missing-msg') { + return { deleted: false as const, stale: false as const } + } + if (input.deletedAt < '2026-07-01T12:00:00.000Z') { + return { deleted: false as const, stale: true as const } + } + return { + deleted: true as const, + stale: false as const, + orphanThreadDeleted: false, + } + }, + ) + const deleteDeliveryEvent = vi.fn(async () => { + throw new Error('do unavailable') + }) + + const { env, idFromName } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvent, + touchThread, + updateMessageDelivery, + setMessageClassification, + deleteMessageMetadata, + deleteDeliveryEvent, + }) + + const thread: EmailThreadRecord = { + id: 'thread-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + subjectNormalized: null, + rootMessageIdHeader: '', + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + } + const message = baseMessage() + + expect( + await mirrorMailboxMessageSnapshot({ + env, + thread, + message, + attachments: [ + { + id: 'att-1', + messageId: 'msg-1', + filename: null, + contentType: null, + contentId: null, + disposition: null, + size: 0, + storageKind: 'unavailable', + storageKey: null, + createdAt: '2026-07-01T12:00:00.000Z', + }, + ], + }), + ).toEqual({ status: 'mirrored' }) + expect(idFromName).toHaveBeenCalledWith('user-aaa') + expect(mirrorMessage).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + thread: { + id: 'thread-1', + inboxId: 'inbox-1', + subjectNormalized: '', + rootMessageIdHeader: '', + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }, + message: expect.objectContaining({ + id: 'msg-1', + fromAddress: 'sender@example.com', + subject: 'Hello', + rawSize: 32, + headers: {}, + }), + attachments: [ + expect.objectContaining({ + id: 'att-1', + contentType: 'application/octet-stream', + storageKind: 'unavailable', + }), + ], + }) + + const detail = { + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + subscriptionEffectState: 'processing', + // Stale JSON copies of promoted columns — must not win. + usageEffectRecordedAt: '2026-01-01T00:00:00.000Z', + usageMonth: 'json-month', + usageBytes: 1, + usageDurationMs: 1, + } + const event: EmailDeliveryEventRecord = { + id: 'evt-1', + messageId: 'msg-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + eventType: 'receive_started', + provider: 'cloudflare-email-routing', + providerMessageId: null, + providerEventId: null, + detailJson: JSON.stringify(detail), + createdAt: '2026-07-02T10:00:00.000Z', + } + expect( + await mirrorMailboxDeliveryEventSnapshot({ + env, + snapshot: { + event, + updatedAt: '2026-07-02T10:00:01.000Z', + needsEffectReconcile: true, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 99, + usageDurationMs: 50, + }, + }), + ).toEqual({ status: 'mirrored' }) + expect(upsertDeliveryEvent).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + event: expect.objectContaining({ + id: 'evt-1', + provider: 'cloudflare-email-routing', + needsEffectReconcile: true, + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + subscriptionEffectState: 'processing', + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 99, + usageDurationMs: 50, + updatedAt: '2026-07-02T10:00:01.000Z', + }), + latestDeliveryStatus: undefined, + }) + + expect( + await mirrorMailboxTouchThread({ + env, + ownerId: 'user-aaa', + threadId: 'thread-1', + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + expect(touchThread).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + threadId: 'thread-1', + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }) + + expect( + await mirrorMailboxUpdateMessageDelivery({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + processingStatus: 'sent', + providerMessageId: 'prov-1', + error: null, + sentAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + + expect( + await mirrorMailboxSetMessageClassification({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + classification: 'quarantined', + classificationReason: 'manual', + updatedAt: '2026-07-01T14:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + + expect( + await mirrorMailboxDeleteMessageMetadata({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + deletedAt: '2026-07-01T15:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + + expect( + await mirrorMailboxDeleteMessageMetadata({ + env, + ownerId: 'user-aaa', + messageId: 'msg-1', + deletedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + + expect( + await mirrorMailboxDeleteMessageMetadata({ + env, + ownerId: 'user-aaa', + messageId: 'missing-msg', + deletedAt: '2026-07-01T15:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + + expect( + await mirrorMailboxDeleteDeliveryEvent({ + env, + ownerId: 'user-aaa', + eventId: 'evt-1', + deletedAt: '2026-07-02T11:00:00.000Z', + }), + ).toEqual({ + status: 'error', + error: expect.objectContaining({ message: 'do unavailable' }), + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-mirror-delete-delivery-event-failed', + expect.objectContaining({ message: 'do unavailable' }), + ) + + expect( + await mirrorMailboxMessageSnapshot({ + env, + message: baseMessage({ userId: systemEmailOwnerId }), + }), + ).toEqual({ status: 'skipped', reason: 'system-email' }) + expect(mirrorMessage).toHaveBeenCalledTimes(1) + + expect( + await mirrorMailboxMessageSnapshot({ + env: {}, + message: baseMessage(), + }), + ).toEqual({ status: 'skipped', reason: 'mailbox-unconfigured' }) +}) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts new file mode 100644 index 0000000000..e27b3174d2 --- /dev/null +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -0,0 +1,261 @@ +import { isSystemEmailOwner } from './email-owner.ts' +import { + mailboxNamespace, + mailboxRpc, + type MailboxEnv, +} from './mailbox-client.ts' +import { + toMailboxAttachmentInput, + toMailboxDeliveryEventInput, + toMailboxMessageInput, + toMailboxThreadInput, + type EmailDeliveryEventMirrorSnapshot, +} from './mailbox-snapshots.ts' +import { + type EmailAttachmentRecord, + type EmailDeliveryStatus, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' +import { + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataInput, + type MailboxDeleteMessageMetadataResult, + type MailboxSetMessageClassificationInput, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, +} from './mailbox-types.ts' + +/** + * Best-effort D1 → Mailbox dual-write helpers. + * + * Never throw into D1 authority paths: catch/log with stable tags and return + * structured `mirrored | stale | skipped | error` outcomes. `system:email` + * stays in D1 only. + */ + +export type MailboxMirrorEnv = MailboxEnv + +export type MailboxMirrorSkipReason = + | 'system-email' + | 'mailbox-unconfigured' + | 'missing-owner' + +export type MailboxMirrorResult = + | { status: 'mirrored' } + | { status: 'stale' } + | { status: 'skipped'; reason: MailboxMirrorSkipReason } + | { status: 'error'; error: unknown } + +function resolveMirrorOwner( + ownerId: string | null | undefined, +): { ok: true; ownerId: string } | { ok: false; result: MailboxMirrorResult } { + if (ownerId == null || ownerId.length === 0) { + return { ok: false, result: { status: 'skipped', reason: 'missing-owner' } } + } + if (isSystemEmailOwner(ownerId)) { + return { + ok: false, + result: { status: 'skipped', reason: 'system-email' }, + } + } + return { ok: true, ownerId } +} + +function skipIfMailboxMissing( + env: MailboxMirrorEnv, +): MailboxMirrorResult | null { + if (!mailboxNamespace(env)) { + return { status: 'skipped', reason: 'mailbox-unconfigured' } + } + return null +} + +function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { + return accepted ? { status: 'mirrored' } : { status: 'stale' } +} + +/** + * Map delete RPC outcome: + * - `deleted: true` → mirrored + * - `deleted: false, stale: true` → stale + * - `deleted: false, stale: false` (missing) → mirrored (idempotent) + */ +function outcomeFromDeleteResult( + result: MailboxDeleteMessageMetadataResult | MailboxDeleteDeliveryEventResult, +): MailboxMirrorResult { + if (result.deleted) return { status: 'mirrored' } + if (result.stale) return { status: 'stale' } + return { status: 'mirrored' } +} + +async function runMirror( + tag: string, + run: () => Promise, +): Promise { + try { + return await run() + } catch (error) { + console.warn(tag, error) + return { status: 'error', error } + } +} + +/** Best-effort full message (+ optional thread/attachments) snapshot mirror. */ +export async function mirrorMailboxMessageSnapshot(input: { + env: MailboxMirrorEnv + thread?: EmailThreadRecord | null + message: EmailMessageRecord + attachments?: Array +}): Promise { + const owner = resolveMirrorOwner(input.message.userId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-message-failed', async () => { + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).mirrorMessage({ + ownerId: owner.ownerId, + thread: input.thread ? toMailboxThreadInput(input.thread) : null, + message: toMailboxMessageInput(input.message), + attachments: input.attachments?.map(toMailboxAttachmentInput), + }) + return outcomeFromAccepted(result.accepted) + }) +} + +/** Best-effort delivery-event snapshot mirror. */ +export async function mirrorMailboxDeliveryEventSnapshot(input: { + env: MailboxMirrorEnv + snapshot: EmailDeliveryEventMirrorSnapshot + latestDeliveryStatus?: { + messageId: string + deliveryStatus: EmailDeliveryStatus + deliveryStatusAt: string + } | null +}): Promise { + const owner = resolveMirrorOwner(input.snapshot.event.userId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-delivery-event-failed', async () => { + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).upsertDeliveryEvent({ + ownerId: owner.ownerId, + event: toMailboxDeliveryEventInput(input.snapshot), + latestDeliveryStatus: input.latestDeliveryStatus, + }) + return outcomeFromAccepted(result.accepted) + }) +} + +/** Best-effort thread touch mirror. */ +export async function mirrorMailboxTouchThread( + input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-touch-thread-failed', async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).touchThread(rpcInput) + return outcomeFromAccepted(result.accepted) + }) +} + +/** Best-effort outbound delivery/processing update mirror. */ +export async function mirrorMailboxUpdateMessageDelivery( + input: MailboxUpdateMessageDeliveryInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror( + 'mailbox-mirror-update-message-delivery-failed', + async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).updateMessageDelivery(rpcInput) + return outcomeFromAccepted(result.accepted) + }, + ) +} + +/** Best-effort inbound classification update mirror. */ +export async function mirrorMailboxSetMessageClassification( + input: MailboxSetMessageClassificationInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror( + 'mailbox-mirror-set-message-classification-failed', + async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).setMessageClassification(rpcInput) + return outcomeFromAccepted(result.accepted) + }, + ) +} + +/** Best-effort metadata-only message delete mirror (no R2). */ +export async function mirrorMailboxDeleteMessageMetadata( + input: MailboxDeleteMessageMetadataInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror( + 'mailbox-mirror-delete-message-metadata-failed', + async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteMessageMetadata(rpcInput) + return outcomeFromDeleteResult(result) + }, + ) +} + +/** Best-effort delivery-event delete mirror. */ +export async function mirrorMailboxDeleteDeliveryEvent( + input: MailboxDeleteDeliveryEventInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-delete-delivery-event-failed', async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteDeliveryEvent(rpcInput) + return outcomeFromDeleteResult(result) + }) +} diff --git a/packages/worker/src/email/mailbox-mutations.ts b/packages/worker/src/email/mailbox-mutations.ts new file mode 100644 index 0000000000..af139f7cbc --- /dev/null +++ b/packages/worker/src/email/mailbox-mutations.ts @@ -0,0 +1,228 @@ +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxClassification, + assertMailboxNonEmptyString, + assertMailboxProcessingStatus, + assertOptionalMailboxCanonicalIsoTimestamp, + type MailboxAcceptedResult, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataResult, + type MailboxSetMessageClassificationInput, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, +} from './mailbox-types.ts' + +/** + * Owner-bound partial mutation helpers for Mailbox SQLite metadata. + * No R2 / alarm side effects — callers own transactions and retention. + */ + +export function touchMailboxThread( + sql: SqlStorage, + input: Omit, +): MailboxAcceptedResult { + const threadId = assertMailboxNonEmptyString(input.threadId, 'threadId') + const lastMessageAt = assertMailboxCanonicalIsoTimestamp( + input.lastMessageAt, + 'lastMessageAt', + ) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + input.updatedAt, + 'updatedAt', + ) + const cursor = sql.exec( + `UPDATE email_threads + SET last_message_at = CASE + WHEN last_message_at < ? THEN ? + ELSE last_message_at + END, + updated_at = ? + WHERE id = ? + AND updated_at <= ?`, + lastMessageAt, + lastMessageAt, + updatedAt, + threadId, + updatedAt, + ) + return { accepted: cursor.rowsWritten > 0 } +} + +export function updateMailboxMessageDelivery( + sql: SqlStorage, + input: Omit, +): MailboxAcceptedResult { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const processingStatus = assertMailboxProcessingStatus(input.processingStatus) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + input.updatedAt, + 'updatedAt', + ) + const sentAt = assertOptionalMailboxCanonicalIsoTimestamp( + input.sentAt, + 'sentAt', + ) + const cursor = sql.exec( + `UPDATE email_messages + SET processing_status = ?, + provider_message_id = ?, + error = ?, + sent_at = ?, + updated_at = ? + WHERE id = ? + AND updated_at <= ?`, + processingStatus, + input.providerMessageId, + input.error, + sentAt, + updatedAt, + messageId, + updatedAt, + ) + return { accepted: cursor.rowsWritten > 0 } +} + +export function setMailboxMessageClassification( + sql: SqlStorage, + input: Omit, +): MailboxAcceptedResult { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const classification = assertMailboxClassification(input.classification) + const updatedAt = assertMailboxCanonicalIsoTimestamp( + input.updatedAt, + 'updatedAt', + ) + const cursor = sql.exec( + `UPDATE email_messages + SET classification = ?, + classification_reason = ?, + updated_at = ? + WHERE id = ? + AND updated_at <= ?`, + classification, + input.classificationReason, + updatedAt, + messageId, + updatedAt, + ) + return { accepted: cursor.rowsWritten > 0 } +} + +/** + * Delete message + attachment metadata and any orphaned thread. Never deletes + * R2 objects. Distinguishes missing (idempotent) from stale (newer retained). + */ +export function deleteMailboxMessageMetadata( + sql: SqlStorage, + input: { messageId: string; deletedAt: string }, +): MailboxDeleteMessageMetadataResult { + const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') + const deletedAt = assertMailboxCanonicalIsoTimestamp( + input.deletedAt, + 'deletedAt', + ) + const existing = sql + .exec<{ thread_id: string | null; updated_at: string }>( + `SELECT thread_id, updated_at FROM email_messages + WHERE id = ? + LIMIT 1`, + messageId, + ) + .toArray()[0] + if (existing == null) { + return { deleted: false, stale: false } + } + if (existing.updated_at > deletedAt) { + return { deleted: false, stale: true } + } + + sql.exec(`DELETE FROM email_attachments WHERE message_id = ?`, messageId) + const messageDelete = sql.exec( + `DELETE FROM email_messages + WHERE id = ? + AND updated_at <= ?`, + messageId, + deletedAt, + ) + if (messageDelete.rowsWritten === 0) { + const stillThere = sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_messages WHERE id = ? LIMIT 1`, + messageId, + ) + .toArray()[0] + return stillThere == null + ? { deleted: false, stale: false } + : { deleted: false, stale: true } + } + + const threadId = existing.thread_id + if (threadId == null || threadId.length === 0) { + return { deleted: true, stale: false, orphanThreadDeleted: false } + } + + const orphanDelete = sql.exec( + `DELETE FROM email_threads + WHERE id = ? + AND NOT EXISTS ( + SELECT 1 FROM email_messages + WHERE thread_id = ? + )`, + threadId, + threadId, + ) + return { + deleted: true, + stale: false, + orphanThreadDeleted: orphanDelete.rowsWritten > 0, + } +} + +/** + * Delete a delivery-event row. SELECT `updated_at` first so missing and stale + * are distinguishable; missing is idempotent success for best-effort callers. + */ +export function deleteMailboxDeliveryEvent( + sql: SqlStorage, + input: { eventId: string; deletedAt: string }, +): MailboxDeleteDeliveryEventResult { + const eventId = assertMailboxNonEmptyString(input.eventId, 'eventId') + const deletedAt = assertMailboxCanonicalIsoTimestamp( + input.deletedAt, + 'deletedAt', + ) + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_delivery_events + WHERE id = ? + LIMIT 1`, + eventId, + ) + .toArray()[0] + if (existing == null) { + return { deleted: false, stale: false } + } + if (existing.updated_at > deletedAt) { + return { deleted: false, stale: true } + } + + const cursor = sql.exec( + `DELETE FROM email_delivery_events + WHERE id = ? + AND updated_at <= ?`, + eventId, + deletedAt, + ) + if (cursor.rowsWritten === 0) { + const stillThere = sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, + eventId, + ) + .toArray()[0] + return stillThere == null + ? { deleted: false, stale: false } + : { deleted: false, stale: true } + } + return { deleted: true, stale: false } +} diff --git a/packages/worker/src/email/mailbox-mutations.workers.test.ts b/packages/worker/src/email/mailbox-mutations.workers.test.ts new file mode 100644 index 0000000000..721617ff1b --- /dev/null +++ b/packages/worker/src/email/mailbox-mutations.workers.test.ts @@ -0,0 +1,357 @@ +import { env } from 'cloudflare:workers' +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' +import { type Mailbox } from './mailbox-do.ts' +import { + assertMailboxThrows, + baseAttachment, + baseDeliveryEvent, + baseMessage, + baseThread, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' + +test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection', async () => { + silenceIncidentalRuntimeWarnings() + const ownerA = uniqueUserId('mut-a') + const ownerB = uniqueUserId('mut-b') + const mailbox = rpcFor(ownerA) + const stub = stubFor(ownerA) + + const thread = baseThread({ + id: 'mut-thread', + lastMessageAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + const message = baseMessage(ownerA, { + id: 'mut-msg', + threadId: thread.id, + processingStatus: 'stored', + classification: 'accepted', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + await mailbox.mirrorMessage({ + ownerId: ownerA, + thread, + message, + attachments: [baseAttachment(ownerA, message.id, { id: 'mut-att' })], + }) + + await runInDurableObject(stub, async (instance: Mailbox) => { + await assertMailboxThrows(/ownerId mismatch/, () => + instance.touchThread({ + ownerId: ownerB, + threadId: thread.id, + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.updateMessageDelivery({ + ownerId: ownerB, + messageId: message.id, + processingStatus: 'sent', + providerMessageId: 'prov', + error: null, + sentAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.setMessageClassification({ + ownerId: ownerB, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'spam', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.deleteMessageMetadata({ + ownerId: ownerB, + messageId: message.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.deleteDeliveryEvent({ + ownerId: ownerB, + eventId: 'missing', + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ) + }) + + const staleTouch = await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T14:00:00.000Z', + updatedAt: '2026-07-01T11:00:00.000Z', + }) + expect(staleTouch).toEqual({ accepted: false }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const regressTouch = await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-06-01T00:00:00.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }) + expect(regressTouch).toEqual({ accepted: true }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }) + + const advanceTouch = await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T15:00:00.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }) + expect(advanceTouch).toEqual({ accepted: true }) + expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ + lastMessageAt: '2026-07-01T15:00:00.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }) + + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: 'missing-thread', + lastMessageAt: '2026-07-01T16:00:00.000Z', + updatedAt: '2026-07-01T16:00:00.000Z', + }), + ).toEqual({ accepted: false }) + + const staleDelivery = await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'failed', + providerMessageId: 'stale-prov', + error: 'stale', + sentAt: null, + updatedAt: '2026-07-01T11:00:00.000Z', + }) + expect(staleDelivery).toEqual({ accepted: false }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + processingStatus: 'stored', + providerMessageId: null, + error: null, + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const deliveryUpdate = await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'sent', + providerMessageId: 'prov-1', + error: null, + sentAt: '2026-07-01T12:05:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + expect(deliveryUpdate).toEqual({ accepted: true }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + processingStatus: 'sent', + providerMessageId: 'prov-1', + sentAt: '2026-07-01T12:05:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const staleClass = await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'stale', + updatedAt: '2026-06-01T00:00:00.000Z', + }) + expect(staleClass).toEqual({ accepted: false }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + classification: 'accepted', + classificationReason: null, + }) + + const classUpdate = await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'Sender matched quarantine rule.', + updatedAt: '2026-07-01T12:00:03.000Z', + }) + expect(classUpdate).toEqual({ accepted: true }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + classification: 'quarantined', + classificationReason: 'Sender matched quarantine rule.', + updatedAt: '2026-07-01T12:00:03.000Z', + }) +}) + +test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no R2', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('del-msg') + const mailbox = rpcFor(userId) + + const thread = baseThread({ id: 'del-thread' }) + const alone = baseMessage(userId, { + id: 'del-alone', + threadId: thread.id, + updatedAt: '2026-07-01T12:00:00.000Z', + }) + const attachment = baseAttachment(userId, alone.id, { id: 'del-att' }) + await mailbox.mirrorMessage({ + ownerId: userId, + thread, + message: alone, + attachments: [attachment], + }) + + const rawKey = emailRawMimeKey(userId, alone.id) + const attKey = emailAttachmentBlobKey(userId, alone.id, attachment.id) + await env.EMAIL_BLOBS.put(rawKey, 'raw-bytes') + await env.EMAIL_BLOBS.put(attKey, 'att-bytes') + + const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) + let blobDeleteCalls = 0 + env.EMAIL_BLOBS.delete = ((keys: string | Array) => { + blobDeleteCalls += 1 + return originalDelete(keys) + }) as typeof env.EMAIL_BLOBS.delete + + try { + const staleDelete = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T11:00:00.000Z', + }) + expect(staleDelete).toEqual({ deleted: false, stale: true }) + expect(await mailbox.getMessage({ messageId: alone.id })).not.toBeNull() + expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: alone.id }), + ).toHaveLength(1) + + const deleted = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T12:00:00.000Z', + }) + expect(deleted).toEqual({ + deleted: true, + stale: false, + orphanThreadDeleted: true, + }) + expect(await mailbox.getMessage({ messageId: alone.id })).toBeNull() + expect(await mailbox.getThread({ threadId: thread.id })).toBeNull() + expect( + await mailbox.listAttachmentsForMessage({ messageId: alone.id }), + ).toHaveLength(0) + expect(await env.EMAIL_BLOBS.get(rawKey)).not.toBeNull() + expect(await env.EMAIL_BLOBS.get(attKey)).not.toBeNull() + expect(blobDeleteCalls).toBe(0) + + const again = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }) + // Missing (`!deleted && !stale`): idempotent success for best-effort callers. + expect(again).toEqual({ deleted: false, stale: false }) + expect(blobDeleteCalls).toBe(0) + } finally { + env.EMAIL_BLOBS.delete = originalDelete + await env.EMAIL_BLOBS.delete(rawKey) + await env.EMAIL_BLOBS.delete(attKey) + } + + const sharedThread = baseThread({ id: 'shared-thread' }) + const keep = baseMessage(userId, { + id: 'keep-msg', + threadId: sharedThread.id, + updatedAt: '2026-07-02T10:00:00.000Z', + }) + const drop = baseMessage(userId, { + id: 'drop-msg', + threadId: sharedThread.id, + updatedAt: '2026-07-02T10:00:00.000Z', + }) + await mailbox.mirrorMessage({ + ownerId: userId, + thread: sharedThread, + message: keep, + }) + await mailbox.mirrorMessage({ + ownerId: userId, + message: drop, + }) + const dropOnly = await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: drop.id, + deletedAt: '2026-07-02T10:00:00.000Z', + }) + expect(dropOnly).toEqual({ + deleted: true, + stale: false, + orphanThreadDeleted: false, + }) + expect(await mailbox.getMessage({ messageId: keep.id })).not.toBeNull() + expect(await mailbox.getThread({ threadId: sharedThread.id })).not.toBeNull() +}) + +test('Mailbox deleteDeliveryEvent: stale, deleted, and missing outcomes', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('del-evt') + const mailbox = rpcFor(userId) + + const message = baseMessage(userId, { + id: 'evt-msg', + direction: 'outbound', + processingStatus: 'sent', + rawMimeKey: null, + }) + await mailbox.mirrorMessage({ ownerId: userId, message }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: baseDeliveryEvent({ + id: 'evt-del', + messageId: message.id, + eventType: 'delivered', + provider: 'cloudflare-email', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + }) + + const staleDelete = await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T09:00:00.000Z', + }) + expect(staleDelete).toEqual({ deleted: false, stale: true }) + expect( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), + ).toHaveLength(1) + + const deleted = await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T10:00:00.000Z', + }) + expect(deleted).toEqual({ deleted: true, stale: false }) + expect( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), + ).toHaveLength(0) + + const again = await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T11:00:00.000Z', + }) + expect(again).toEqual({ deleted: false, stale: false }) +}) diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts new file mode 100644 index 0000000000..1cc170a450 --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -0,0 +1,152 @@ +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import { recordMailboxParityEvent } from './mailbox-parity-events.ts' + +test('mailbox parity writes compact indexed mirror and parity data points', () => { + const writeDataPoint = vi.fn() + const env = { + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, + } + + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'mirror_message', + outcome: 'mirrored', + timestamp: '2026-07-30T10:00:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'touch_thread', + outcome: 'stale', + timestamp: '2026-07-30T10:05:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'set_message_classification', + outcome: 'skipped', + timestamp: '2026-07-30T10:06:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'delete_message_metadata', + outcome: 'error', + timestamp: '2026-07-30T10:07:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-2', + category: 'parity', + operation: 'compare_messages', + outcome: 'mismatch', + d1Count: 12, + doCount: 10, + timestamp: '2026-07-30T11:00:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-3', + category: 'parity', + operation: 'compare_threads', + outcome: 'match', + timestamp: '2026-07-30T12:00:00.000Z', + }) + + expect(writeDataPoint).toHaveBeenCalledTimes(6) + expect(writeDataPoint).toHaveBeenNthCalledWith(1, { + indexes: ['user-1'], + blobs: ['mirror_message', 'mirrored', '2026-07-30T10:00:00.000Z'], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(2, { + indexes: ['user-1'], + blobs: ['touch_thread', 'stale', '2026-07-30T10:05:00.000Z'], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(3, { + indexes: ['user-1'], + blobs: [ + 'set_message_classification', + 'skipped', + '2026-07-30T10:06:00.000Z', + ], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(4, { + indexes: ['user-1'], + blobs: ['delete_message_metadata', 'error', '2026-07-30T10:07:00.000Z'], + doubles: [0, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(5, { + indexes: ['user-2'], + blobs: ['compare_messages', 'mismatch', '2026-07-30T11:00:00.000Z'], + doubles: [2, 1], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(6, { + indexes: ['user-3'], + blobs: ['compare_threads', 'match', '2026-07-30T12:00:00.000Z'], + doubles: [0, 1], + }) +}) + +test('mailbox parity skips system email and never throws when the binding is absent or fails', () => { + const writeDataPoint = vi.fn() + consoleWarn.mockImplementation(() => {}) + + recordMailboxParityEvent( + { + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, + }, + { + userId: systemEmailOwnerId, + category: 'mirror', + operation: 'mirror_message', + outcome: 'mirrored', + }, + ) + expect(writeDataPoint).not.toHaveBeenCalled() + + expect(() => + recordMailboxParityEvent( + {}, + { + userId: 'user-1', + category: 'mirror', + operation: 'update_message_delivery', + outcome: 'mirrored', + }, + ), + ).not.toThrow() + + expect(() => + recordMailboxParityEvent( + { + EMAIL_EVENTS: { + writeDataPoint() { + throw new Error('analytics unavailable') + }, + } as unknown as AnalyticsEngineDataset, + }, + { + userId: 'user-1', + category: 'parity', + operation: 'compare_delivery_events', + outcome: 'mismatch', + d1Count: 3, + doCount: 1, + }, + ), + ).not.toThrow() + + expect(consoleWarn).toHaveBeenCalledTimes(1) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-event-write-failed', + expect.objectContaining({ message: 'analytics unavailable' }), + ) +}) diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts new file mode 100644 index 0000000000..e3dcd74229 --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -0,0 +1,87 @@ +import { isSystemEmailOwner } from './email-owner.ts' + +export type MailboxParityEnv = { + EMAIL_EVENTS?: AnalyticsEngineDataset +} + +export type MailboxMirrorOperation = + | 'mirror_message' + | 'upsert_delivery_event' + | 'touch_thread' + | 'update_message_delivery' + | 'set_message_classification' + | 'delete_message_metadata' + | 'delete_delivery_event' + +export type MailboxMirrorOutcome = 'mirrored' | 'stale' | 'skipped' | 'error' + +export type MailboxParityOperation = + | 'compare_threads' + | 'compare_messages' + | 'compare_attachments' + | 'compare_delivery_events' + +export type MailboxParityOutcome = 'match' | 'mismatch' + +export type MailboxParityEvent = + | { + userId: string + category: 'mirror' + operation: MailboxMirrorOperation + outcome: MailboxMirrorOutcome + timestamp?: string + } + | { + userId: string + category: 'parity' + operation: MailboxParityOperation + outcome: MailboxParityOutcome + d1Count?: number + doCount?: number + timestamp?: string + } + +function resolveCountDelta(event: MailboxParityEvent): number { + if ( + event.category === 'parity' && + event.d1Count !== undefined && + event.doCount !== undefined + ) { + return event.d1Count - event.doCount + } + return 0 +} + +/** + * Additive Analytics Engine schema (same EMAIL_EVENTS dataset as reporting): + * + * - index1: stable user id (per-user isolation) + * - blob1: mirror or parity operation + * - blob2: operation outcome + * - blob3: source event timestamp (ISO 8601) + * - double1: D1-minus-DO count delta (0 when absent or not applicable) + * - double2: event weight (always 1) + * + * `system:email` is excluded — operator mail stays in D1 only. + */ +export function recordMailboxParityEvent( + env: MailboxParityEnv, + event: MailboxParityEvent, +): void { + try { + if (!env.EMAIL_EVENTS || !event.userId) return + if (isSystemEmailOwner(event.userId)) return + + env.EMAIL_EVENTS.writeDataPoint({ + indexes: [event.userId], + blobs: [ + event.operation, + event.outcome, + event.timestamp ?? new Date().toISOString(), + ], + doubles: [resolveCountDelta(event), 1], + }) + } catch (error) { + console.warn('mailbox-parity-event-write-failed', error) + } +} diff --git a/packages/worker/src/email/mailbox-snapshots.node.test.ts b/packages/worker/src/email/mailbox-snapshots.node.test.ts new file mode 100644 index 0000000000..bd26855c74 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshots.node.test.ts @@ -0,0 +1,309 @@ +import { expect, test } from 'vitest' +import { + type EmailAttachmentRecord, + type EmailDeliveryEventRecord, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' +import { + toMailboxAttachmentInput, + toMailboxDeliveryEventInput, + toMailboxMessageInput, + toMailboxThreadInput, + type EmailDeliveryEventMirrorSnapshot, +} from './mailbox-snapshots.ts' + +test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted columns', () => { + const thread: EmailThreadRecord = { + id: 'thread-1', + userId: 'user-1', + inboxId: 'inbox-1', + subjectNormalized: null, + rootMessageIdHeader: null, + lastMessageAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:30:00.000Z', + } + expect(toMailboxThreadInput(thread)).toEqual({ + id: 'thread-1', + inboxId: 'inbox-1', + subjectNormalized: '', + rootMessageIdHeader: null, + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:30:00.000Z', + }) + + const message: EmailMessageRecord = { + id: 'msg-1', + direction: 'inbound', + userId: 'user-1', + inboxId: null, + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: null, + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: null, + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: null, + authResults: null, + textBody: 'hello', + htmlBody: null, + rawMimeKey: 'user-1/msg-1.eml', + rawSize: null, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + } + expect(toMailboxMessageInput(message)).toEqual({ + id: 'msg-1', + direction: 'inbound', + inboxId: null, + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: '', + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: '', + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: {}, + authResults: null, + textBody: 'hello', + htmlBody: null, + rawMimeKey: 'user-1/msg-1.eml', + rawSize: 0, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }) + + const attachment: EmailAttachmentRecord = { + id: 'att-1', + messageId: 'msg-1', + filename: 'note.txt', + contentType: null, + contentId: null, + disposition: null, + size: 12, + storageKind: 'external', + storageKey: 'user-1/msg-1/att-1', + createdAt: '2026-07-01T12:00:00.000Z', + } + expect(toMailboxAttachmentInput(attachment)).toEqual({ + id: 'att-1', + messageId: 'msg-1', + filename: 'note.txt', + contentType: 'application/octet-stream', + contentId: null, + disposition: null, + size: 12, + storageKind: 'external', + storageKey: 'user-1/msg-1/att-1', + createdAt: '2026-07-01T12:00:00.000Z', + }) + + // detail_json still owns inbound/effect lease fields, but also may contain + // stale copies of promoted columns — converter must prefer D1 columns. + const detail = { + state: 'received', + fingerprint: 'fp-1', + storageLease: 'lease-1', + storageLeaseAt: '2026-07-02T10:00:00.000Z', + cleanupLease: 'cleanup-1', + cleanupLeaseAt: '2026-07-02T10:05:00.000Z', + cleanupRetryAt: '2026-07-02T11:00:00.000Z', + expectedAttachmentCount: 2, + finalizationToken: 'token-1', + reconcileAfter: '2026-07-02T12:00:00.000Z', + dedupeExpiresAt: '2026-07-04T10:00:00.000Z', + usageEffectRecordedAt: '2026-07-02T09:00:00.000Z', + usageEffectSuppressedAt: null, + usageStartedAt: '2026-07-02T09:59:00.000Z', + usageMonth: 'stale-from-json', + usageBytes: 1, + usageDurationMs: 1, + usageEffectRetryAt: null, + usageEffectLease: 'usage-lease', + usageEffectLeaseAt: '2026-07-02T10:00:30.000Z', + subscriptionEffectState: 'pending', + subscriptionEffectLease: 'sub-lease', + subscriptionEffectLeaseAt: '2026-07-02T10:02:00.000Z', + subscriptionEffectRetryAt: '2026-07-02T13:00:00.000Z', + subscriptionEffectAttemptCount: 1, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + recipient: 'owner@example.com', + } + const event: EmailDeliveryEventRecord = { + id: 'evt-1', + messageId: 'msg-1', + userId: 'user-1', + inboxId: 'inbox-1', + eventType: 'received', + provider: null, + providerMessageId: null, + providerEventId: 'provider-evt-1', + detailJson: JSON.stringify(detail), + createdAt: '2026-07-02T10:00:00.000Z', + } + const snapshot: EmailDeliveryEventMirrorSnapshot = { + event, + updatedAt: '2026-07-02T10:00:01.000Z', + needsEffectReconcile: true, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + } + expect(toMailboxDeliveryEventInput(snapshot)).toEqual({ + id: 'evt-1', + messageId: 'msg-1', + inboxId: 'inbox-1', + eventType: 'received', + provider: 'kody', + providerMessageId: null, + providerEventId: 'provider-evt-1', + detailJson: event.detailJson, + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-1', + storageLease: 'lease-1', + storageLeaseAt: '2026-07-02T10:00:00.000Z', + cleanupLease: 'cleanup-1', + cleanupLeaseAt: '2026-07-02T10:05:00.000Z', + cleanupRetryAt: '2026-07-02T11:00:00.000Z', + expectedAttachmentCount: 2, + finalizationToken: 'token-1', + reconcileAfter: '2026-07-02T12:00:00.000Z', + dedupeExpiresAt: '2026-07-04T10:00:00.000Z', + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageEffectSuppressedAt: null, + usageStartedAt: '2026-07-02T09:59:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + usageEffectRetryAt: null, + usageEffectLease: 'usage-lease', + usageEffectLeaseAt: '2026-07-02T10:00:30.000Z', + subscriptionEffectState: 'pending', + subscriptionEffectLease: 'sub-lease', + subscriptionEffectLeaseAt: '2026-07-02T10:02:00.000Z', + subscriptionEffectRetryAt: '2026-07-02T13:00:00.000Z', + subscriptionEffectAttemptCount: 1, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }) +}) + +test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or enums', () => { + const baseEvent: EmailDeliveryEventRecord = { + id: 'evt-bad', + messageId: null, + userId: 'user-1', + inboxId: null, + eventType: 'received', + provider: 'kody', + providerMessageId: null, + providerEventId: null, + detailJson: '{}', + createdAt: '2026-07-02T10:00:00.000Z', + } + const baseSnapshot = { + updatedAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: false, + usageEffectRecordedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + } satisfies Omit + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { ...baseEvent, detailJson: '{not-json' }, + }), + ).toThrow(/detailJson is not valid JSON/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { ...baseEvent, detailJson: '[]' }, + }), + ).toThrow(/detailJson must be a JSON object/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { + ...baseEvent, + detailJson: JSON.stringify({ state: 'not-a-state' }), + }, + }), + ).toThrow(/detail\.state is invalid/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { + ...baseEvent, + detailJson: JSON.stringify({ + subscriptionEffectState: 'bogus', + }), + }, + }), + ).toThrow(/detail\.subscriptionEffectState is invalid/) + + expect(() => + toMailboxDeliveryEventInput({ + ...baseSnapshot, + event: { + ...baseEvent, + detailJson: JSON.stringify({ expectedAttachmentCount: 'lots' }), + }, + }), + ).toThrow(/detail\.expectedAttachmentCount must be a finite number/) + + expect(() => + toMailboxAttachmentInput({ + id: 'att-bad', + messageId: 'msg-1', + filename: null, + contentType: 'text/plain', + contentId: null, + disposition: null, + size: 1, + storageKind: 'not-a-kind', + storageKey: null, + createdAt: '2026-07-01T12:00:00.000Z', + }), + ).toThrow(/storageKind is invalid/) +}) diff --git a/packages/worker/src/email/mailbox-snapshots.ts b/packages/worker/src/email/mailbox-snapshots.ts new file mode 100644 index 0000000000..f2bd487315 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshots.ts @@ -0,0 +1,362 @@ +import { + type EmailAttachmentRecord, + type EmailDeliveryEventRecord, + type EmailMessageRecord, + type EmailThreadRecord, +} from './types.ts' +import { + assertMailboxClassification, + assertMailboxDeliveryEventType, + assertMailboxDeliveryStatus, + assertMailboxDirection, + assertMailboxProcessingStatus, + assertMailboxStorageKind, + mailboxInboundDeliveryStateValues, + mailboxSubscriptionEffectStateValues, + type MailboxAttachmentInput, + type MailboxDeliveryEventInput, + type MailboxMessageInput, + type MailboxThreadInput, +} from './mailbox-types.ts' + +/** + * Pure D1 → Mailbox complete-snapshot converters. + * + * Normalize nulls to Mailbox SQLite defaults (empty strings, `[]`, `{}`, `0`, + * `application/octet-stream`, `kody`). Delivery-event conversion takes an + * additive {@link EmailDeliveryEventMirrorSnapshot}: D1-promoted columns are + * authoritative; `detail_json` supplies only fields that remain JSON-owned. + * Invalid persisted JSON/enums fail clearly rather than fabricating state. + */ + +/** + * Additive delivery-event mirror input. D1 `EmailDeliveryEventRecord` does not + * yet expose promoted columns or `updated_at`; callers pass those explicitly. + * + * Authoritative D1 columns (do not read these from `detail_json`): + * - `needs_effect_reconcile` + * - `usage_effect_recorded_at` + * - `usage_month` + * - `usage_bytes` + * - `usage_duration_ms` + * + * Still JSON-owned in `detail_json` (inbound/effect lease fields, etc.). + */ +export type EmailDeliveryEventMirrorSnapshot = { + event: EmailDeliveryEventRecord + /** D1 delivery events lack `updated_at`; required for stale rejection. */ + updatedAt: string + /** D1 `needs_effect_reconcile` column. */ + needsEffectReconcile: boolean + /** D1 `usage_effect_recorded_at` column. */ + usageEffectRecordedAt: string | null + /** D1 `usage_month` column. */ + usageMonth: string | null + /** D1 `usage_bytes` column. */ + usageBytes: number | null + /** D1 `usage_duration_ms` column. */ + usageDurationMs: number | null +} + +function requireOptionalString(value: unknown, label: string): string | null { + if (value == null) return null + if (typeof value !== 'string') { + throw new Error( + `Mailbox snapshot ${label} must be a string or null; got ${JSON.stringify(value)}.`, + ) + } + return value +} + +function requireOptionalNumber(value: unknown, label: string): number | null { + if (value == null) return null + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error( + `Mailbox snapshot ${label} must be a finite number or null; got ${JSON.stringify(value)}.`, + ) + } + return value +} + +function requireOptionalEnum( + values: ReadonlyArray, + value: unknown, + label: string, +): T | null { + if (value == null) return null + if ( + typeof value !== 'string' || + !(values as ReadonlyArray).includes(value) + ) { + throw new Error( + `Mailbox snapshot ${label} is invalid; got ${JSON.stringify(value)}.`, + ) + } + return value as T +} + +function parseDetailRecord(detailJson: string): Record { + let parsed: unknown + try { + parsed = JSON.parse(detailJson) as unknown + } catch { + throw new Error( + `Mailbox snapshot detailJson is not valid JSON; got ${JSON.stringify(detailJson)}.`, + ) + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error( + `Mailbox snapshot detailJson must be a JSON object; got ${JSON.stringify(parsed)}.`, + ) + } + return parsed as Record +} + +/** Convert a D1 thread row into a full Mailbox thread snapshot. */ +export function toMailboxThreadInput( + thread: EmailThreadRecord, +): MailboxThreadInput { + return { + id: thread.id, + inboxId: thread.inboxId, + subjectNormalized: thread.subjectNormalized ?? '', + rootMessageIdHeader: thread.rootMessageIdHeader, + lastMessageAt: thread.lastMessageAt ?? thread.createdAt, + createdAt: thread.createdAt, + updatedAt: thread.updatedAt, + } +} + +/** Convert a D1 message row into a full Mailbox message snapshot. */ +export function toMailboxMessageInput( + message: EmailMessageRecord, +): MailboxMessageInput { + const deliveryStatus = + message.deliveryStatus == null + ? null + : assertMailboxDeliveryStatus(message.deliveryStatus) + return { + id: message.id, + direction: assertMailboxDirection(message.direction), + inboxId: message.inboxId, + threadId: message.threadId, + senderIdentityId: message.senderIdentityId, + fromAddress: message.fromAddress ?? '', + envelopeFrom: message.envelopeFrom, + toAddresses: message.toAddresses ?? [], + ccAddresses: message.ccAddresses ?? [], + bccAddresses: message.bccAddresses ?? [], + replyToAddresses: message.replyToAddresses ?? [], + subject: message.subject ?? '', + messageIdHeader: message.messageIdHeader, + inReplyToHeader: message.inReplyToHeader, + references: message.references ?? [], + headers: message.headers ?? {}, + authResults: message.authResults, + textBody: message.textBody, + htmlBody: message.htmlBody, + rawMimeKey: message.rawMimeKey, + rawSize: message.rawSize ?? 0, + processingStatus: assertMailboxProcessingStatus(message.processingStatus), + classification: assertMailboxClassification(message.classification), + classificationReason: message.classificationReason, + providerMessageId: message.providerMessageId, + deliveryStatus, + deliveryStatusAt: message.deliveryStatusAt, + error: message.error, + receivedAt: message.receivedAt, + sentAt: message.sentAt, + createdAt: message.createdAt, + updatedAt: message.updatedAt, + } +} + +/** Convert a D1 attachment row into a full Mailbox attachment snapshot. */ +export function toMailboxAttachmentInput( + attachment: EmailAttachmentRecord, +): MailboxAttachmentInput { + return { + id: attachment.id, + messageId: attachment.messageId, + filename: attachment.filename, + contentType: attachment.contentType ?? 'application/octet-stream', + contentId: attachment.contentId, + disposition: attachment.disposition, + size: attachment.size, + storageKind: assertMailboxStorageKind(attachment.storageKind), + storageKey: attachment.storageKey, + createdAt: attachment.createdAt, + } +} + +/** + * Convert a D1 delivery-event mirror snapshot into a full Mailbox input. + * Promoted D1 columns win; remaining inbound/effect fields come from JSON. + */ +export function toMailboxDeliveryEventInput( + snapshot: EmailDeliveryEventMirrorSnapshot, +): MailboxDeliveryEventInput { + const { + event, + updatedAt, + needsEffectReconcile, + usageEffectRecordedAt, + usageMonth, + usageBytes, + usageDurationMs, + } = snapshot + if (typeof needsEffectReconcile !== 'boolean') { + throw new Error( + `Mailbox snapshot needsEffectReconcile must be a boolean; got ${JSON.stringify(needsEffectReconcile)}.`, + ) + } + if (typeof updatedAt !== 'string' || updatedAt.length === 0) { + throw new Error( + `Mailbox snapshot updatedAt must be a non-empty string; got ${JSON.stringify(updatedAt)}.`, + ) + } + if ( + usageEffectRecordedAt != null && + typeof usageEffectRecordedAt !== 'string' + ) { + throw new Error( + `Mailbox snapshot usageEffectRecordedAt must be a string or null; got ${JSON.stringify(usageEffectRecordedAt)}.`, + ) + } + if (usageMonth != null && typeof usageMonth !== 'string') { + throw new Error( + `Mailbox snapshot usageMonth must be a string or null; got ${JSON.stringify(usageMonth)}.`, + ) + } + if ( + usageBytes != null && + (typeof usageBytes !== 'number' || !Number.isFinite(usageBytes)) + ) { + throw new Error( + `Mailbox snapshot usageBytes must be a finite number or null; got ${JSON.stringify(usageBytes)}.`, + ) + } + if ( + usageDurationMs != null && + (typeof usageDurationMs !== 'number' || !Number.isFinite(usageDurationMs)) + ) { + throw new Error( + `Mailbox snapshot usageDurationMs must be a finite number or null; got ${JSON.stringify(usageDurationMs)}.`, + ) + } + + const detail = parseDetailRecord(event.detailJson) + + return { + id: event.id, + messageId: event.messageId, + inboxId: event.inboxId, + eventType: assertMailboxDeliveryEventType(event.eventType), + provider: event.provider ?? 'kody', + providerMessageId: event.providerMessageId, + providerEventId: event.providerEventId, + detailJson: event.detailJson, + needsEffectReconcile, + state: requireOptionalEnum( + mailboxInboundDeliveryStateValues, + detail['state'], + 'detail.state', + ), + fingerprint: requireOptionalString( + detail['fingerprint'], + 'detail.fingerprint', + ), + storageLease: requireOptionalString( + detail['storageLease'], + 'detail.storageLease', + ), + storageLeaseAt: requireOptionalString( + detail['storageLeaseAt'], + 'detail.storageLeaseAt', + ), + cleanupLease: requireOptionalString( + detail['cleanupLease'], + 'detail.cleanupLease', + ), + cleanupLeaseAt: requireOptionalString( + detail['cleanupLeaseAt'], + 'detail.cleanupLeaseAt', + ), + cleanupRetryAt: requireOptionalString( + detail['cleanupRetryAt'], + 'detail.cleanupRetryAt', + ), + expectedAttachmentCount: requireOptionalNumber( + detail['expectedAttachmentCount'], + 'detail.expectedAttachmentCount', + ), + finalizationToken: requireOptionalString( + detail['finalizationToken'], + 'detail.finalizationToken', + ), + reconcileAfter: requireOptionalString( + detail['reconcileAfter'], + 'detail.reconcileAfter', + ), + dedupeExpiresAt: requireOptionalString( + detail['dedupeExpiresAt'], + 'detail.dedupeExpiresAt', + ), + // Authoritative D1 columns — ignore any JSON copies of these keys. + usageEffectRecordedAt, + usageEffectSuppressedAt: requireOptionalString( + detail['usageEffectSuppressedAt'], + 'detail.usageEffectSuppressedAt', + ), + usageStartedAt: requireOptionalString( + detail['usageStartedAt'], + 'detail.usageStartedAt', + ), + usageMonth, + usageBytes, + usageDurationMs, + usageEffectRetryAt: requireOptionalString( + detail['usageEffectRetryAt'], + 'detail.usageEffectRetryAt', + ), + usageEffectLease: requireOptionalString( + detail['usageEffectLease'], + 'detail.usageEffectLease', + ), + usageEffectLeaseAt: requireOptionalString( + detail['usageEffectLeaseAt'], + 'detail.usageEffectLeaseAt', + ), + subscriptionEffectState: requireOptionalEnum( + mailboxSubscriptionEffectStateValues, + detail['subscriptionEffectState'], + 'detail.subscriptionEffectState', + ), + subscriptionEffectLease: requireOptionalString( + detail['subscriptionEffectLease'], + 'detail.subscriptionEffectLease', + ), + subscriptionEffectLeaseAt: requireOptionalString( + detail['subscriptionEffectLeaseAt'], + 'detail.subscriptionEffectLeaseAt', + ), + subscriptionEffectRetryAt: requireOptionalString( + detail['subscriptionEffectRetryAt'], + 'detail.subscriptionEffectRetryAt', + ), + subscriptionEffectAttemptCount: requireOptionalNumber( + detail['subscriptionEffectAttemptCount'], + 'detail.subscriptionEffectAttemptCount', + ), + subscriptionEffectDeadLetterAt: requireOptionalString( + detail['subscriptionEffectDeadLetterAt'], + 'detail.subscriptionEffectDeadLetterAt', + ), + subscriptionEffectLastError: requireOptionalString( + detail['subscriptionEffectLastError'], + 'detail.subscriptionEffectLastError', + ), + createdAt: event.createdAt, + updatedAt, + } +} diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 75b38f9927..3d9ea8b629 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -11,10 +11,13 @@ import { encodeMailboxListCursor, mailboxNowIso, normalizeMailboxPageSize, + type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, + type MailboxDeleteDeliveryEventResult, + type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, @@ -22,9 +25,19 @@ import { type MailboxMessageInput, type MailboxMessageRecord, type MailboxSearchMessagesInput, + type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, + type MailboxTouchThreadInput, + type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' +import { + deleteMailboxDeliveryEvent, + deleteMailboxMessageMetadata, + setMailboxMessageClassification, + touchMailboxThread, + updateMailboxMessageDelivery, +} from './mailbox-mutations.ts' import { type EmailDeliveryEventType, type EmailDeliveryStatus, @@ -441,6 +454,38 @@ export class MailboxStore { return deliveryEventOwnsMessage(this.sql, eventId, messageId) } + touchThread( + input: Omit, + ): MailboxAcceptedResult { + return touchMailboxThread(this.sql, input) + } + + updateMessageDelivery( + input: Omit, + ): MailboxAcceptedResult { + return updateMailboxMessageDelivery(this.sql, input) + } + + setMessageClassification( + input: Omit, + ): MailboxAcceptedResult { + return setMailboxMessageClassification(this.sql, input) + } + + deleteMessageMetadata(input: { + messageId: string + deletedAt: string + }): MailboxDeleteMessageMetadataResult { + return deleteMailboxMessageMetadata(this.sql, input) + } + + deleteDeliveryEvent(input: { + eventId: string + deletedAt: string + }): MailboxDeleteDeliveryEventResult { + return deleteMailboxDeliveryEvent(this.sql, input) + } + getThread(threadId: string): MailboxThreadRecord | null { const row = this.sql .exec>( diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 6e281c850c..6178ce59b6 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -353,6 +353,84 @@ export type MailboxExportPhase = | 'attachment' | 'delivery_event' +/** Partial thread touch — equal/newer `updatedAt` only; never regresses `lastMessageAt`. */ +export type MailboxTouchThreadInput = { + ownerId: string + threadId: string + lastMessageAt: string + updatedAt: string +} + +/** Partial delivery/processing update — equal/newer `updatedAt` only. */ +export type MailboxUpdateMessageDeliveryInput = { + ownerId: string + messageId: string + processingStatus: EmailProcessingStatus + providerMessageId: string | null + error: string | null + sentAt: string | null + updatedAt: string +} + +/** Partial classification update — equal/newer `updatedAt` only. */ +export type MailboxSetMessageClassificationInput = { + ownerId: string + messageId: string + classification: EmailClassification + classificationReason: string | null + updatedAt: string +} + +/** + * Metadata-only message delete. Never touches R2. Stale when the row's + * `updated_at` is newer than `deletedAt`. Idempotent when already absent. + */ +export type MailboxDeleteMessageMetadataInput = { + ownerId: string + messageId: string + deletedAt: string +} + +/** + * Metadata-only delivery-event delete. Stale when the row's `updated_at` is + * newer than `deletedAt`. Idempotent when already absent. + */ +export type MailboxDeleteDeliveryEventInput = { + ownerId: string + eventId: string + deletedAt: string +} + +export type MailboxAcceptedResult = { + accepted: boolean +} + +/** + * Message metadata delete outcome. + * - `deleted: true` — row removed; includes `orphanThreadDeleted` + * - `deleted: false, stale: false` — missing (idempotent success) + * - `deleted: false, stale: true` — newer row retained + * + * Best-effort callers map missing (`!deleted && !stale`) to success. + */ +export type MailboxDeleteMessageMetadataResult = + | { deleted: true; stale: false; orphanThreadDeleted: boolean } + | { deleted: false; stale: true } + | { deleted: false; stale: false } + +/** + * Delivery-event delete outcome. + * - `deleted: true` — row removed + * - `deleted: false, stale: false` — missing (idempotent success) + * - `deleted: false, stale: true` — newer row retained + * + * Best-effort callers map missing (`!deleted && !stale`) to success. + */ +export type MailboxDeleteDeliveryEventResult = + | { deleted: true; stale: false } + | { deleted: false; stale: true } + | { deleted: false; stale: false } + export type MailboxRpc = { mirrorMessage: (input: { /** @@ -382,6 +460,21 @@ export type MailboxRpc = { accepted: boolean updatedLatestStatus: boolean }> + touchThread: ( + input: MailboxTouchThreadInput, + ) => Promise + updateMessageDelivery: ( + input: MailboxUpdateMessageDeliveryInput, + ) => Promise + setMessageClassification: ( + input: MailboxSetMessageClassificationInput, + ) => Promise + deleteMessageMetadata: ( + input: MailboxDeleteMessageMetadataInput, + ) => Promise + deleteDeliveryEvent: ( + input: MailboxDeleteDeliveryEventInput, + ) => Promise getThread: (input: { threadId: string }) => Promise diff --git a/packages/worker/src/email/system-email.ts b/packages/worker/src/email/system-email.ts index 73484a2052..f373e17aff 100644 --- a/packages/worker/src/email/system-email.ts +++ b/packages/worker/src/email/system-email.ts @@ -10,9 +10,10 @@ import { getEmailInboxByName, } from './repo.ts' import { buildPlatformEmailAddress } from './platform-address.ts' +import { systemEmailOwnerId } from './email-owner.ts' import { type EmailInboxAddressRecord, type EmailInboxRecord } from './types.ts' -export const systemEmailOwnerId = 'system:email' +export { systemEmailOwnerId } export const systemEmailLocals = [ 'kody', From 861524852bf9a9d074a49fb6e60e5c344a75af8f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 09:24:23 +0000 Subject: [PATCH 05/62] fix(email): align Mailbox mutation parity Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 84 +++-- packages/worker/src/email/mailbox-do.ts | 121 ++++--- .../src/email/mailbox-mirror.node.test.ts | 123 ++++--- packages/worker/src/email/mailbox-mirror.ts | 100 ++++-- .../worker/src/email/mailbox-mutations.ts | 178 +++++----- .../email/mailbox-mutations.workers.test.ts | 314 +++++++++++------- .../email/mailbox-parity-events.node.test.ts | 40 ++- .../worker/src/email/mailbox-parity-events.ts | 19 +- .../email/mailbox-snapshot-repo.node.test.ts | 120 +++++++ .../worker/src/email/mailbox-snapshot-repo.ts | 126 +++++++ .../src/email/mailbox-snapshots.node.test.ts | 53 ++- .../worker/src/email/mailbox-snapshots.ts | 136 +++----- packages/worker/src/email/mailbox-store.ts | 45 --- packages/worker/src/email/mailbox-types.ts | 64 ++-- 14 files changed, 946 insertions(+), 577 deletions(-) create mode 100644 packages/worker/src/email/mailbox-snapshot-repo.node.test.ts create mode 100644 packages/worker/src/email/mailbox-snapshot-repo.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e7c38ddfcf..402bf79a48 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -383,20 +383,22 @@ defined in `packages/worker/src/email/mailbox-parity-events.ts` and reconciliation — they do not feed the admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) -- `blob1` — mirror or parity operation name +- `blob1` — namespaced event type (`mailbox_mirror:` or + `mailbox_parity:`) so unfiltered consumers can distinguish these + from `email_send` / `email_receive` / `email_delivery` - `blob2` — operation outcome - `blob3` — source event timestamp (ISO 8601) -- `double1` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; +- `double1` — event weight (always `1`, matching reporting) +- `double2` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; `0` when absent or not applicable) -- `double2` — event weight (always `1`) Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, `update_message_delivery`, `set_message_classification`, -`delete_message_metadata`, `delete_delivery_event`. Mirror outcomes: `mirrored`, -`stale`, `skipped`, `error`. Parity operations: `compare_threads`, -`compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity -outcomes: `match`, `mismatch`. Writes are best-effort and never throw into D1 -authority paths. +`delete_message_metadata`, `delete_delivery_event`, `delete_thread_if_empty`. +Mirror outcomes: `mirrored`, `stale`, `skipped`, `error`. Parity operations: +`compare_threads`, `compare_messages`, `compare_attachments`, +`compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Writes are +best-effort and never throw into D1 authority paths. Two D1 reporting projections deliberately remain: @@ -663,19 +665,23 @@ external attachment `storageKey` values against the canonical builders for that `ownerId`. **Partial mutation RPCs** (phase 2a; owner-bound, monotonic on `updatedAt`, no -R2 or alarm side effects beyond the usual retention dirty mark): +R2 or retention-alarm side effects — unlike full snapshot mirrors, these do not +mark retention dirty or reschedule alarms): - `touchThread` — advance `last_message_at` / `updated_at` without a full snapshot; `last_message_at` never moves backward - `updateMessageDelivery` — outbound processing fields (`processing_status`, `provider_message_id`, `error`, `sent_at`) - `setMessageClassification` — inbound classification fields -- `deleteMessageMetadata` — metadata-only delete (attachments + message + orphan - thread); never deletes R2; distinguishes missing (idempotent) from stale - (newer `updated_at` retained) -- `deleteDeliveryEvent` — metadata-only delivery-event delete with the same - missing vs stale semantics - +- `deleteMessageMetadata` — metadata-only delete (null delivery-event + `message_id`, then attachments + message); never deletes R2 or empty threads +- `deleteDeliveryEvent` — metadata-only delivery-event delete +- `deleteThreadIfEmpty` — deferred empty-thread cleanup (D1 + `deleteEmptyEmailThreads` parity); stale-safe by `thread.updated_at` + +Partial touch/update/classify RPCs return `accepted`, `missing` (target absent — +idempotent for best-effort callers), or `stale` (newer `updated_at` retained). +Delete RPCs return `deleted`, `missing`, or `stale` with the same semantics. Implementation lives in `mailbox-mutations.ts` (SQLite helpers) and is exposed through `mailbox-do.ts` RPCs. All require `ownerId` and reject cross-owner calls. @@ -686,16 +692,20 @@ lightweight module exporting `systemEmailOwnerId` (`'system:email'`) and dual-write helpers can skip the reserved operator inbox without import cycles. Operator mail is never mirrored into per-user Mailbox objects. -**D1 → Mailbox snapshot adapters** (`mailbox-snapshots.ts`): pure converters -that turn D1 rows into complete Mailbox wire inputs — `toMailboxThreadInput`, -`toMailboxMessageInput`, `toMailboxAttachmentInput`, and -`toMailboxDeliveryEventInput`. They normalize nulls to Mailbox SQLite defaults -(empty strings, `[]`, `{}`, `0`, `application/octet-stream`, `kody`) and fail -clearly on invalid persisted JSON/enums rather than fabricating state. Delivery -events take an additive `EmailDeliveryEventMirrorSnapshot`: D1-promoted columns -(`needs_effect_reconcile`, usage-effect fields) are authoritative; remaining -inbound/effect lease fields come from `detail_json`. D1 delivery events lack -`updated_at`; callers pass it explicitly for stale rejection. +**D1 → Mailbox snapshot adapters** (`mailbox-snapshots.ts`; delivery loads in +`mailbox-snapshot-repo.ts`): pure converters that turn D1 rows into complete +Mailbox wire inputs — `toMailboxThreadInput`, `toMailboxMessageInput`, +`toMailboxAttachmentInput`, and `toMailboxDeliveryEventInput`. They normalize +nulls to Mailbox SQLite defaults (empty strings, `[]`, `{}`, `0`, +`application/octet-stream`, `kody`) and fail clearly on invalid persisted +JSON/enums rather than fabricating state. Delivery events consume a complete +`EmailDeliveryEventMirrorProjection` loaded as one cohesive D1 row (base columns +plus promoted `needs_effect_reconcile` and usage-effect fields); inbound/effect +lease fields still come from `detail_json`. Callers load via +`getMailboxDeliveryEventMirrorInput` / `getEmailDeliveryEventMirrorProjection` +and supply only `sourceMutationAt` — the canonical mirror `updatedAt` from the +D1 mutation (inserts use `created_at`). Callers must not stitch promoted columns +field-by-field. **Best-effort mirror helpers** (`mailbox-mirror.ts`): non-throwing wrappers around the DO RPCs for future D1-authoritative dual-write. Each returns a @@ -704,13 +714,14 @@ structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, `mailbox-unconfigured` | `missing-owner`), or `{ status: 'error', error }`. Failures log with stable tags (for example `mailbox-mirror-message-failed`) and never propagate into D1 commit paths. Helpers cover full snapshots -(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot`) and -partial mutations (`mirrorMailboxTouchThread`, -`mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, -`mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`). -**There are no live dual-write callers in the phase-2a PR** — only unit/worker -tests invoke these helpers. D1 remains sole authority for all live mail -read/write paths. +(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot` — prefer +loading delivery events with `getMailboxDeliveryEventMirrorInput`) and partial +mutations (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, +`mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, +`mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). **There +are no live dual-write callers in the phase-2a PR** — only unit/worker tests +invoke these helpers. D1 remains sole authority for all live mail read/write +paths. **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -752,10 +763,11 @@ surface that later live-path wiring will call. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO completeness against D1. D1 stays authoritative for reads. Phase 2a (current - PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, `mailbox-mirror.ts`, - `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, - `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, - `deleteDeliveryEvent`), and `mailbox-parity-events.ts`. Those helpers and + PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, + `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-mutations.ts` + + matching DO RPCs (`touchThread`, `updateMessageDelivery`, + `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, + `deleteThreadIfEmpty`), and `mailbox-parity-events.ts`. Those helpers and parity-event writers are library-only today — **no production path invokes them yet**; live-path dual-write and parity wiring remain for the rest of phase 2. diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 5222063275..c8b06ce2f4 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -13,23 +13,31 @@ import { selectMailboxRetentionWriteAlarm, } from './mailbox-retention.ts' import { MailboxStore } from './mailbox-store.ts' +import { + deleteMailboxDeliveryEvent, + deleteMailboxMessageMetadata, + deleteMailboxThreadIfEmpty, + setMailboxMessageClassification, + touchMailboxThread, + updateMailboxMessageDelivery, +} from './mailbox-mutations.ts' import { assertMailboxNonEmptyString, - type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, type MailboxDeleteDeliveryEventInput, - type MailboxDeleteDeliveryEventResult, type MailboxDeleteMessageMetadataInput, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, type MailboxListMessagesInput, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxPartialMutationResult, type MailboxRpc, type MailboxSearchMessagesInput, type MailboxSetMessageClassificationInput, @@ -230,19 +238,14 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async touchThread( input: MailboxTouchThreadInput, - ): Promise { - let accepted = false + ): Promise { + let result: MailboxPartialMutationResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - accepted = this.store.touchThread({ - threadId: input.threadId, - lastMessageAt: input.lastMessageAt, - updatedAt: input.updatedAt, - }).accepted + const { ownerId: _ownerId, ...mutationInput } = input + result = touchMailboxThread(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() - return { accepted } + return result } /** @@ -251,22 +254,14 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async updateMessageDelivery( input: MailboxUpdateMessageDeliveryInput, - ): Promise { - let accepted = false + ): Promise { + let result: MailboxPartialMutationResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - accepted = this.store.updateMessageDelivery({ - messageId: input.messageId, - processingStatus: input.processingStatus, - providerMessageId: input.providerMessageId, - error: input.error, - sentAt: input.sentAt, - updatedAt: input.updatedAt, - }).accepted + const { ownerId: _ownerId, ...mutationInput } = input + result = updateMailboxMessageDelivery(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() - return { accepted } + return result } /** @@ -275,42 +270,32 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async setMessageClassification( input: MailboxSetMessageClassificationInput, - ): Promise { - let accepted = false + ): Promise { + let result: MailboxPartialMutationResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - accepted = this.store.setMessageClassification({ - messageId: input.messageId, - classification: input.classification, - classificationReason: input.classificationReason, - updatedAt: input.updatedAt, - }).accepted + const { ownerId: _ownerId, ...mutationInput } = input + result = setMailboxMessageClassification( + this.ctx.storage.sql, + mutationInput, + ) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() - return { accepted } + return result } /** - * Metadata-only delete (attachments + message + orphan thread). Never - * deletes R2. Stale when the message `updated_at` is newer than `deletedAt`. + * Metadata-only delete (null delivery-event message_id + attachments + + * message). Never deletes R2 or empty threads. */ async deleteMessageMetadata( input: MailboxDeleteMessageMetadataInput, - ): Promise { - let result: MailboxDeleteMessageMetadataResult = { - deleted: false, - stale: false, - } + ): Promise { + let result: MailboxDeleteResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - result = this.store.deleteMessageMetadata({ - messageId: input.messageId, - deletedAt: input.deletedAt, - }) + const { ownerId: _ownerId, ...mutationInput } = input + result = deleteMailboxMessageMetadata(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() return result } @@ -320,20 +305,28 @@ class MailboxBase extends DurableObject implements MailboxRpc { */ async deleteDeliveryEvent( input: MailboxDeleteDeliveryEventInput, - ): Promise { - let result: MailboxDeleteDeliveryEventResult = { - deleted: false, - stale: false, - } + ): Promise { + let result: MailboxDeleteResult = { status: 'missing' } this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - result = this.store.deleteDeliveryEvent({ - eventId: input.eventId, - deletedAt: input.deletedAt, - }) + const { ownerId: _ownerId, ...mutationInput } = input + result = deleteMailboxDeliveryEvent(this.ctx.storage.sql, mutationInput) + }) + return result + } + + /** + * Deferred empty-thread cleanup. Stale-safe by `thread.updated_at`. + */ + async deleteThreadIfEmpty( + input: MailboxDeleteThreadIfEmptyInput, + ): Promise { + let result: MailboxDeleteResult = { status: 'missing' } + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + const { ownerId: _ownerId, ...mutationInput } = input + result = deleteMailboxThreadIfEmpty(this.ctx.storage.sql, mutationInput) }) - this.markRetentionDirty() - await this.ensureRetentionAlarm() return result } @@ -437,22 +430,22 @@ export { mailboxMessageRetentionDays, mailboxRetentionContinuationDelayMs, mailboxRetentionRetryDelayMs, - type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReference, type MailboxBlobReferencePage, type MailboxCountResult, type MailboxDeleteDeliveryEventInput, - type MailboxDeleteDeliveryEventResult, type MailboxDeleteMessageMetadataInput, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, type MailboxExportRow, type MailboxMessageInput, type MailboxMessageRecord, + type MailboxPartialMutationResult, type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 4668e5bb64..b7bc8344e0 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -5,16 +5,14 @@ import { mirrorMailboxDeliveryEventSnapshot, mirrorMailboxDeleteDeliveryEvent, mirrorMailboxDeleteMessageMetadata, + mirrorMailboxDeleteThreadIfEmpty, mirrorMailboxMessageSnapshot, mirrorMailboxSetMessageClassification, mirrorMailboxTouchThread, mirrorMailboxUpdateMessageDelivery, } from './mailbox-mirror.ts' -import { - type EmailDeliveryEventRecord, - type EmailMessageRecord, - type EmailThreadRecord, -} from './types.ts' +import { type EmailMessageRecord, type EmailThreadRecord } from './types.ts' +import { type MailboxDeliveryEventInput } from './mailbox-types.ts' function fakeMailboxEnv(stub: Record) { const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) @@ -84,27 +82,30 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep accepted: true, updatedLatestStatus: false, })) - const touchThread = vi.fn(async () => ({ accepted: true })) - const updateMessageDelivery = vi.fn(async () => ({ accepted: false })) - const setMessageClassification = vi.fn(async () => ({ accepted: true })) + const touchThread = vi.fn(async () => ({ status: 'accepted' as const })) + const updateMessageDelivery = vi.fn(async () => ({ + status: 'stale' as const, + })) + const setMessageClassification = vi.fn(async () => ({ + status: 'missing' as const, + })) const deleteMessageMetadata = vi.fn( async (input: { messageId: string; deletedAt: string }) => { if (input.messageId === 'missing-msg') { - return { deleted: false as const, stale: false as const } + return { status: 'missing' as const } } if (input.deletedAt < '2026-07-01T12:00:00.000Z') { - return { deleted: false as const, stale: true as const } - } - return { - deleted: true as const, - stale: false as const, - orphanThreadDeleted: false, + return { status: 'stale' as const } } + return { status: 'deleted' as const } }, ) const deleteDeliveryEvent = vi.fn(async () => { throw new Error('do unavailable') }) + const deleteThreadIfEmpty = vi.fn(async () => ({ + status: 'missing' as const, + })) const { env, idFromName } = fakeMailboxEnv({ mirrorMessage, @@ -114,6 +115,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep setMessageClassification, deleteMessageMetadata, deleteDeliveryEvent, + deleteThreadIfEmpty, }) const thread: EmailThreadRecord = { @@ -177,59 +179,61 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep ], }) - const detail = { - state: 'storing', - fingerprint: 'fp-mirror', - storageLease: 'lease', - subscriptionEffectState: 'processing', - // Stale JSON copies of promoted columns — must not win. - usageEffectRecordedAt: '2026-01-01T00:00:00.000Z', - usageMonth: 'json-month', - usageBytes: 1, - usageDurationMs: 1, - } - const event: EmailDeliveryEventRecord = { + const deliveryEvent: MailboxDeliveryEventInput = { id: 'evt-1', messageId: 'msg-1', - userId: 'user-aaa', inboxId: 'inbox-1', eventType: 'receive_started', provider: 'cloudflare-email-routing', providerMessageId: null, providerEventId: null, - detailJson: JSON.stringify(detail), + detailJson: JSON.stringify({ + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + subscriptionEffectState: 'processing', + }), + needsEffectReconcile: true, + state: 'storing', + fingerprint: 'fp-mirror', + storageLease: 'lease', + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: '2026-07', + usageBytes: 99, + usageDurationMs: 50, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: 'processing', + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', } expect( await mirrorMailboxDeliveryEventSnapshot({ env, - snapshot: { - event, - updatedAt: '2026-07-02T10:00:01.000Z', - needsEffectReconcile: true, - usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', - usageMonth: '2026-07', - usageBytes: 99, - usageDurationMs: 50, - }, + ownerId: 'user-aaa', + event: deliveryEvent, }), ).toEqual({ status: 'mirrored' }) expect(upsertDeliveryEvent).toHaveBeenCalledWith({ ownerId: 'user-aaa', - event: expect.objectContaining({ - id: 'evt-1', - provider: 'cloudflare-email-routing', - needsEffectReconcile: true, - state: 'storing', - fingerprint: 'fp-mirror', - storageLease: 'lease', - subscriptionEffectState: 'processing', - usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', - usageMonth: '2026-07', - usageBytes: 99, - usageDurationMs: 50, - updatedAt: '2026-07-02T10:00:01.000Z', - }), + event: deliveryEvent, latestDeliveryStatus: undefined, }) @@ -262,6 +266,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'stale' }) + // missing partial mutation is idempotent success for best-effort dual-write expect( await mirrorMailboxSetMessageClassification({ env, @@ -300,6 +305,20 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'mirrored' }) + expect( + await mirrorMailboxDeleteThreadIfEmpty({ + env, + ownerId: 'user-aaa', + threadId: 'thread-1', + deletedAt: '2026-07-01T16:00:00.000Z', + }), + ).toEqual({ status: 'mirrored' }) + expect(deleteThreadIfEmpty).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + threadId: 'thread-1', + deletedAt: '2026-07-01T16:00:00.000Z', + }) + expect( await mirrorMailboxDeleteDeliveryEvent({ env, diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index e27b3174d2..b72b4f4fe3 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -6,10 +6,8 @@ import { } from './mailbox-client.ts' import { toMailboxAttachmentInput, - toMailboxDeliveryEventInput, toMailboxMessageInput, toMailboxThreadInput, - type EmailDeliveryEventMirrorSnapshot, } from './mailbox-snapshots.ts' import { type EmailAttachmentRecord, @@ -19,9 +17,11 @@ import { } from './types.ts' import { type MailboxDeleteDeliveryEventInput, - type MailboxDeleteDeliveryEventResult, type MailboxDeleteMessageMetadataInput, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, + type MailboxDeliveryEventInput, + type MailboxPartialMutationResult, type MailboxSetMessageClassificationInput, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, @@ -33,6 +33,10 @@ import { * Never throw into D1 authority paths: catch/log with stable tags and return * structured `mirrored | stale | skipped | error` outcomes. `system:email` * stays in D1 only. + * + * Delivery-event snapshots should be loaded via + * `getMailboxDeliveryEventMirrorInput` (mailbox-snapshot-repo) so callers do + * not supply promoted D1 columns individually. */ export type MailboxMirrorEnv = MailboxEnv @@ -72,22 +76,46 @@ function skipIfMailboxMissing( return null } -function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { - return accepted ? { status: 'mirrored' } : { status: 'stale' } +/** Map touch/update/classify: accepted/missing → mirrored; stale → stale. */ +function outcomeFromPartialMutation( + result: MailboxPartialMutationResult, +): MailboxMirrorResult { + switch (result.status) { + case 'accepted': + case 'missing': + return { status: 'mirrored' } + case 'stale': + return { status: 'stale' } + default: { + const exhaustive: never = result + throw new Error( + `Unhandled mailbox partial mutation status: ${JSON.stringify(exhaustive)}`, + ) + } + } } -/** - * Map delete RPC outcome: - * - `deleted: true` → mirrored - * - `deleted: false, stale: true` → stale - * - `deleted: false, stale: false` (missing) → mirrored (idempotent) - */ +/** Map delete RPC: deleted/missing → mirrored; stale → stale. */ function outcomeFromDeleteResult( - result: MailboxDeleteMessageMetadataResult | MailboxDeleteDeliveryEventResult, + result: MailboxDeleteResult, ): MailboxMirrorResult { - if (result.deleted) return { status: 'mirrored' } - if (result.stale) return { status: 'stale' } - return { status: 'mirrored' } + switch (result.status) { + case 'deleted': + case 'missing': + return { status: 'mirrored' } + case 'stale': + return { status: 'stale' } + default: { + const exhaustive: never = result + throw new Error( + `Unhandled mailbox delete status: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { + return accepted ? { status: 'mirrored' } : { status: 'stale' } } async function runMirror( @@ -128,17 +156,24 @@ export async function mirrorMailboxMessageSnapshot(input: { }) } -/** Best-effort delivery-event snapshot mirror. */ +/** + * Best-effort delivery-event snapshot mirror. + * + * Prefer loading `event` with `getMailboxDeliveryEventMirrorInput` so promoted + * D1 columns and detail decoding stay cohesive. `event.updatedAt` must be the + * canonical `sourceMutationAt` from that load (inserts: `created_at`). + */ export async function mirrorMailboxDeliveryEventSnapshot(input: { env: MailboxMirrorEnv - snapshot: EmailDeliveryEventMirrorSnapshot + ownerId: string + event: MailboxDeliveryEventInput latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus deliveryStatusAt: string } | null }): Promise { - const owner = resolveMirrorOwner(input.snapshot.event.userId) + const owner = resolveMirrorOwner(input.ownerId) if (!owner.ok) return owner.result const skippedMailbox = skipIfMailboxMissing(input.env) if (skippedMailbox) return skippedMailbox @@ -149,7 +184,7 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { userId: owner.ownerId, }).upsertDeliveryEvent({ ownerId: owner.ownerId, - event: toMailboxDeliveryEventInput(input.snapshot), + event: input.event, latestDeliveryStatus: input.latestDeliveryStatus, }) return outcomeFromAccepted(result.accepted) @@ -171,7 +206,7 @@ export async function mirrorMailboxTouchThread( env: input.env, userId: owner.ownerId, }).touchThread(rpcInput) - return outcomeFromAccepted(result.accepted) + return outcomeFromPartialMutation(result) }) } @@ -192,7 +227,7 @@ export async function mirrorMailboxUpdateMessageDelivery( env: input.env, userId: owner.ownerId, }).updateMessageDelivery(rpcInput) - return outcomeFromAccepted(result.accepted) + return outcomeFromPartialMutation(result) }, ) } @@ -214,7 +249,7 @@ export async function mirrorMailboxSetMessageClassification( env: input.env, userId: owner.ownerId, }).setMessageClassification(rpcInput) - return outcomeFromAccepted(result.accepted) + return outcomeFromPartialMutation(result) }, ) } @@ -259,3 +294,22 @@ export async function mirrorMailboxDeleteDeliveryEvent( return outcomeFromDeleteResult(result) }) } + +/** Best-effort empty-thread cleanup mirror (D1 deleteEmptyEmailThreads parity). */ +export async function mirrorMailboxDeleteThreadIfEmpty( + input: MailboxDeleteThreadIfEmptyInput & { env: MailboxMirrorEnv }, +): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) return owner.result + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) return skippedMailbox + + return runMirror('mailbox-mirror-delete-thread-if-empty-failed', async () => { + const { env: _env, ...rpcInput } = input + const result = await mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteThreadIfEmpty(rpcInput) + return outcomeFromDeleteResult(result) + }) +} diff --git a/packages/worker/src/email/mailbox-mutations.ts b/packages/worker/src/email/mailbox-mutations.ts index af139f7cbc..039985d632 100644 --- a/packages/worker/src/email/mailbox-mutations.ts +++ b/packages/worker/src/email/mailbox-mutations.ts @@ -4,9 +4,9 @@ import { assertMailboxNonEmptyString, assertMailboxProcessingStatus, assertOptionalMailboxCanonicalIsoTimestamp, - type MailboxAcceptedResult, - type MailboxDeleteDeliveryEventResult, - type MailboxDeleteMessageMetadataResult, + type MailboxDeleteResult, + type MailboxDeleteThreadIfEmptyInput, + type MailboxPartialMutationResult, type MailboxSetMessageClassificationInput, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, @@ -14,13 +14,13 @@ import { /** * Owner-bound partial mutation helpers for Mailbox SQLite metadata. - * No R2 / alarm side effects — callers own transactions and retention. + * No R2 / alarm side effects — callers own transactions. */ export function touchMailboxThread( sql: SqlStorage, input: Omit, -): MailboxAcceptedResult { +): MailboxPartialMutationResult { const threadId = assertMailboxNonEmptyString(input.threadId, 'threadId') const lastMessageAt = assertMailboxCanonicalIsoTimestamp( input.lastMessageAt, @@ -30,7 +30,16 @@ export function touchMailboxThread( input.updatedAt, 'updatedAt', ) - const cursor = sql.exec( + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_threads WHERE id = ? LIMIT 1`, + threadId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > updatedAt) return { status: 'stale' } + + sql.exec( `UPDATE email_threads SET last_message_at = CASE WHEN last_message_at < ? THEN ? @@ -45,13 +54,13 @@ export function touchMailboxThread( threadId, updatedAt, ) - return { accepted: cursor.rowsWritten > 0 } + return { status: 'accepted' } } export function updateMailboxMessageDelivery( sql: SqlStorage, input: Omit, -): MailboxAcceptedResult { +): MailboxPartialMutationResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const processingStatus = assertMailboxProcessingStatus(input.processingStatus) const updatedAt = assertMailboxCanonicalIsoTimestamp( @@ -62,7 +71,16 @@ export function updateMailboxMessageDelivery( input.sentAt, 'sentAt', ) - const cursor = sql.exec( + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, + messageId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > updatedAt) return { status: 'stale' } + + sql.exec( `UPDATE email_messages SET processing_status = ?, provider_message_id = ?, @@ -79,20 +97,29 @@ export function updateMailboxMessageDelivery( messageId, updatedAt, ) - return { accepted: cursor.rowsWritten > 0 } + return { status: 'accepted' } } export function setMailboxMessageClassification( sql: SqlStorage, input: Omit, -): MailboxAcceptedResult { +): MailboxPartialMutationResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const classification = assertMailboxClassification(input.classification) const updatedAt = assertMailboxCanonicalIsoTimestamp( input.updatedAt, 'updatedAt', ) - const cursor = sql.exec( + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, + messageId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > updatedAt) return { status: 'stale' } + + sql.exec( `UPDATE email_messages SET classification = ?, classification_reason = ?, @@ -105,87 +132,58 @@ export function setMailboxMessageClassification( messageId, updatedAt, ) - return { accepted: cursor.rowsWritten > 0 } + return { status: 'accepted' } } /** - * Delete message + attachment metadata and any orphaned thread. Never deletes - * R2 objects. Distinguishes missing (idempotent) from stale (newer retained). + * Delete message + attachment metadata. Nulls delivery-event `message_id` + * first (D1 parity). Never deletes R2 or empty threads. */ export function deleteMailboxMessageMetadata( sql: SqlStorage, input: { messageId: string; deletedAt: string }, -): MailboxDeleteMessageMetadataResult { +): MailboxDeleteResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const deletedAt = assertMailboxCanonicalIsoTimestamp( input.deletedAt, 'deletedAt', ) const existing = sql - .exec<{ thread_id: string | null; updated_at: string }>( - `SELECT thread_id, updated_at FROM email_messages + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_messages WHERE id = ? LIMIT 1`, messageId, ) .toArray()[0] - if (existing == null) { - return { deleted: false, stale: false } - } - if (existing.updated_at > deletedAt) { - return { deleted: false, stale: true } - } + if (existing == null) return { status: 'missing' } + if (existing.updated_at > deletedAt) return { status: 'stale' } + sql.exec( + `UPDATE email_delivery_events + SET message_id = NULL + WHERE message_id = ?`, + messageId, + ) sql.exec(`DELETE FROM email_attachments WHERE message_id = ?`, messageId) - const messageDelete = sql.exec( + sql.exec( `DELETE FROM email_messages WHERE id = ? AND updated_at <= ?`, messageId, deletedAt, ) - if (messageDelete.rowsWritten === 0) { - const stillThere = sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_messages WHERE id = ? LIMIT 1`, - messageId, - ) - .toArray()[0] - return stillThere == null - ? { deleted: false, stale: false } - : { deleted: false, stale: true } - } - - const threadId = existing.thread_id - if (threadId == null || threadId.length === 0) { - return { deleted: true, stale: false, orphanThreadDeleted: false } - } - - const orphanDelete = sql.exec( - `DELETE FROM email_threads - WHERE id = ? - AND NOT EXISTS ( - SELECT 1 FROM email_messages - WHERE thread_id = ? - )`, - threadId, - threadId, - ) - return { - deleted: true, - stale: false, - orphanThreadDeleted: orphanDelete.rowsWritten > 0, - } + return { status: 'deleted' } } /** * Delete a delivery-event row. SELECT `updated_at` first so missing and stale - * are distinguishable; missing is idempotent success for best-effort callers. + * are distinguishable. */ export function deleteMailboxDeliveryEvent( sql: SqlStorage, input: { eventId: string; deletedAt: string }, -): MailboxDeleteDeliveryEventResult { +): MailboxDeleteResult { const eventId = assertMailboxNonEmptyString(input.eventId, 'eventId') const deletedAt = assertMailboxCanonicalIsoTimestamp( input.deletedAt, @@ -199,30 +197,54 @@ export function deleteMailboxDeliveryEvent( eventId, ) .toArray()[0] - if (existing == null) { - return { deleted: false, stale: false } - } - if (existing.updated_at > deletedAt) { - return { deleted: false, stale: true } - } + if (existing == null) return { status: 'missing' } + if (existing.updated_at > deletedAt) return { status: 'stale' } - const cursor = sql.exec( + sql.exec( `DELETE FROM email_delivery_events WHERE id = ? AND updated_at <= ?`, eventId, deletedAt, ) - if (cursor.rowsWritten === 0) { - const stillThere = sql - .exec<{ ok: number }>( - `SELECT 1 AS ok FROM email_delivery_events WHERE id = ? LIMIT 1`, - eventId, - ) - .toArray()[0] - return stillThere == null - ? { deleted: false, stale: false } - : { deleted: false, stale: true } - } - return { deleted: true, stale: false } + return { status: 'deleted' } +} + +/** + * Delete a thread only when it has no messages. Stale-safe by + * `thread.updated_at`. Not-empty / already-absent → `missing` (idempotent). + */ +export function deleteMailboxThreadIfEmpty( + sql: SqlStorage, + input: Omit, +): MailboxDeleteResult { + const threadId = assertMailboxNonEmptyString(input.threadId, 'threadId') + const deletedAt = assertMailboxCanonicalIsoTimestamp( + input.deletedAt, + 'deletedAt', + ) + const existing = sql + .exec<{ updated_at: string }>( + `SELECT updated_at FROM email_threads + WHERE id = ? + LIMIT 1`, + threadId, + ) + .toArray()[0] + if (existing == null) return { status: 'missing' } + if (existing.updated_at > deletedAt) return { status: 'stale' } + + const cursor = sql.exec( + `DELETE FROM email_threads + WHERE id = ? + AND updated_at <= ? + AND NOT EXISTS ( + SELECT 1 FROM email_messages + WHERE thread_id = ? + )`, + threadId, + deletedAt, + threadId, + ) + return cursor.rowsWritten > 0 ? { status: 'deleted' } : { status: 'missing' } } diff --git a/packages/worker/src/email/mailbox-mutations.workers.test.ts b/packages/worker/src/email/mailbox-mutations.workers.test.ts index 721617ff1b..f36a0bc28a 100644 --- a/packages/worker/src/email/mailbox-mutations.workers.test.ts +++ b/packages/worker/src/email/mailbox-mutations.workers.test.ts @@ -15,7 +15,7 @@ import { uniqueUserId, } from './mailbox-test-helpers.ts' -test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection', async () => { +test('Mailbox mutation RPCs: owner bind, accepted/missing/stale updates', async () => { silenceIncidentalRuntimeWarnings() const ownerA = uniqueUserId('mut-a') const ownerB = uniqueUserId('mut-b') @@ -84,39 +84,49 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' deletedAt: '2026-07-01T13:00:00.000Z', }), ) + await assertMailboxThrows(/ownerId mismatch/, () => + instance.deleteThreadIfEmpty({ + ownerId: ownerB, + threadId: thread.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ) }) - const staleTouch = await mailbox.touchThread({ - ownerId: ownerA, - threadId: thread.id, - lastMessageAt: '2026-07-01T14:00:00.000Z', - updatedAt: '2026-07-01T11:00:00.000Z', - }) - expect(staleTouch).toEqual({ accepted: false }) + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T14:00:00.000Z', + updatedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ lastMessageAt: '2026-07-01T12:00:00.000Z', updatedAt: '2026-07-01T12:00:00.000Z', }) - const regressTouch = await mailbox.touchThread({ - ownerId: ownerA, - threadId: thread.id, - lastMessageAt: '2026-06-01T00:00:00.000Z', - updatedAt: '2026-07-01T12:00:01.000Z', - }) - expect(regressTouch).toEqual({ accepted: true }) + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-06-01T00:00:00.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ lastMessageAt: '2026-07-01T12:00:00.000Z', updatedAt: '2026-07-01T12:00:01.000Z', }) - const advanceTouch = await mailbox.touchThread({ - ownerId: ownerA, - threadId: thread.id, - lastMessageAt: '2026-07-01T15:00:00.000Z', - updatedAt: '2026-07-01T12:00:02.000Z', - }) - expect(advanceTouch).toEqual({ accepted: true }) + expect( + await mailbox.touchThread({ + ownerId: ownerA, + threadId: thread.id, + lastMessageAt: '2026-07-01T15:00:00.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getThread({ threadId: thread.id })).toMatchObject({ lastMessageAt: '2026-07-01T15:00:00.000Z', updatedAt: '2026-07-01T12:00:02.000Z', @@ -129,18 +139,19 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' lastMessageAt: '2026-07-01T16:00:00.000Z', updatedAt: '2026-07-01T16:00:00.000Z', }), - ).toEqual({ accepted: false }) + ).toEqual({ status: 'missing' }) - const staleDelivery = await mailbox.updateMessageDelivery({ - ownerId: ownerA, - messageId: message.id, - processingStatus: 'failed', - providerMessageId: 'stale-prov', - error: 'stale', - sentAt: null, - updatedAt: '2026-07-01T11:00:00.000Z', - }) - expect(staleDelivery).toEqual({ accepted: false }) + expect( + await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'failed', + providerMessageId: 'stale-prov', + error: 'stale', + sentAt: null, + updatedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ processingStatus: 'stored', providerMessageId: null, @@ -148,16 +159,29 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' updatedAt: '2026-07-01T12:00:00.000Z', }) - const deliveryUpdate = await mailbox.updateMessageDelivery({ - ownerId: ownerA, - messageId: message.id, - processingStatus: 'sent', - providerMessageId: 'prov-1', - error: null, - sentAt: '2026-07-01T12:05:00.000Z', - updatedAt: '2026-07-01T12:00:00.000Z', - }) - expect(deliveryUpdate).toEqual({ accepted: true }) + expect( + await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: 'missing-msg', + processingStatus: 'sent', + providerMessageId: null, + error: null, + sentAt: null, + updatedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) + + expect( + await mailbox.updateMessageDelivery({ + ownerId: ownerA, + messageId: message.id, + processingStatus: 'sent', + providerMessageId: 'prov-1', + error: null, + sentAt: '2026-07-01T12:05:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ processingStatus: 'sent', providerMessageId: 'prov-1', @@ -165,27 +189,25 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' updatedAt: '2026-07-01T12:00:00.000Z', }) - const staleClass = await mailbox.setMessageClassification({ - ownerId: ownerA, - messageId: message.id, - classification: 'quarantined', - classificationReason: 'stale', - updatedAt: '2026-06-01T00:00:00.000Z', - }) - expect(staleClass).toEqual({ accepted: false }) - expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ - classification: 'accepted', - classificationReason: null, - }) + expect( + await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'stale', + updatedAt: '2026-06-01T00:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) - const classUpdate = await mailbox.setMessageClassification({ - ownerId: ownerA, - messageId: message.id, - classification: 'quarantined', - classificationReason: 'Sender matched quarantine rule.', - updatedAt: '2026-07-01T12:00:03.000Z', - }) - expect(classUpdate).toEqual({ accepted: true }) + expect( + await mailbox.setMessageClassification({ + ownerId: ownerA, + messageId: message.id, + classification: 'quarantined', + classificationReason: 'Sender matched quarantine rule.', + updatedAt: '2026-07-01T12:00:03.000Z', + }), + ).toEqual({ status: 'accepted' }) expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ classification: 'quarantined', classificationReason: 'Sender matched quarantine rule.', @@ -193,12 +215,15 @@ test('Mailbox mutation RPCs: owner bind, monotonic updates, and stale rejection' }) }) -test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no R2', async () => { +test('Mailbox deleteMessageMetadata: nulls delivery message_id, no orphan/R2', async () => { silenceIncidentalRuntimeWarnings() const userId = uniqueUserId('del-msg') const mailbox = rpcFor(userId) - const thread = baseThread({ id: 'del-thread' }) + const thread = baseThread({ + id: 'del-thread', + updatedAt: '2026-07-01T12:00:00.000Z', + }) const alone = baseMessage(userId, { id: 'del-alone', threadId: thread.id, @@ -211,6 +236,16 @@ test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no message: alone, attachments: [attachment], }) + await mailbox.upsertDeliveryEvent({ + ownerId: userId, + event: baseDeliveryEvent({ + id: 'del-evt', + messageId: alone.id, + eventType: 'received', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }), + }) const rawKey = emailRawMimeKey(userId, alone.id) const attKey = emailAttachmentBlobKey(userId, alone.id, attachment.id) @@ -225,52 +260,80 @@ test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no }) as typeof env.EMAIL_BLOBS.delete try { - const staleDelete = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: alone.id, - deletedAt: '2026-07-01T11:00:00.000Z', - }) - expect(staleDelete).toEqual({ deleted: false, stale: true }) - expect(await mailbox.getMessage({ messageId: alone.id })).not.toBeNull() - expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() expect( - await mailbox.listAttachmentsForMessage({ messageId: alone.id }), - ).toHaveLength(1) + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + expect(await mailbox.getMessage({ messageId: alone.id })).not.toBeNull() - const deleted = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: alone.id, - deletedAt: '2026-07-01T12:00:00.000Z', - }) - expect(deleted).toEqual({ - deleted: true, - stale: false, - orphanThreadDeleted: true, - }) + expect( + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) expect(await mailbox.getMessage({ messageId: alone.id })).toBeNull() - expect(await mailbox.getThread({ threadId: thread.id })).toBeNull() + // Thread remains for deferred empty-thread cleanup. + expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() expect( await mailbox.listAttachmentsForMessage({ messageId: alone.id }), ).toHaveLength(0) + const events = await mailbox.listDeliveryEvents({ limit: 5 }) + expect(events).toEqual([ + expect.objectContaining({ id: 'del-evt', messageId: null }), + ]) expect(await env.EMAIL_BLOBS.get(rawKey)).not.toBeNull() expect(await env.EMAIL_BLOBS.get(attKey)).not.toBeNull() expect(blobDeleteCalls).toBe(0) - const again = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: alone.id, - deletedAt: '2026-07-01T13:00:00.000Z', - }) - // Missing (`!deleted && !stale`): idempotent success for best-effort callers. - expect(again).toEqual({ deleted: false, stale: false }) + expect( + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: alone.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) expect(blobDeleteCalls).toBe(0) + + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: thread.id, + deletedAt: '2026-07-01T11:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) + expect(await mailbox.getThread({ threadId: thread.id })).not.toBeNull() + + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: thread.id, + deletedAt: '2026-07-01T12:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) + expect(await mailbox.getThread({ threadId: thread.id })).toBeNull() + + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: thread.id, + deletedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) } finally { env.EMAIL_BLOBS.delete = originalDelete await env.EMAIL_BLOBS.delete(rawKey) await env.EMAIL_BLOBS.delete(attKey) } - const sharedThread = baseThread({ id: 'shared-thread' }) + const sharedThread = baseThread({ + id: 'shared-thread', + updatedAt: '2026-07-02T10:00:00.000Z', + }) const keep = baseMessage(userId, { id: 'keep-msg', threadId: sharedThread.id, @@ -290,17 +353,21 @@ test('Mailbox deleteMessageMetadata: stale guard, orphan thread, idempotency, no ownerId: userId, message: drop, }) - const dropOnly = await mailbox.deleteMessageMetadata({ - ownerId: userId, - messageId: drop.id, - deletedAt: '2026-07-02T10:00:00.000Z', - }) - expect(dropOnly).toEqual({ - deleted: true, - stale: false, - orphanThreadDeleted: false, - }) + expect( + await mailbox.deleteMessageMetadata({ + ownerId: userId, + messageId: drop.id, + deletedAt: '2026-07-02T10:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) expect(await mailbox.getMessage({ messageId: keep.id })).not.toBeNull() + expect( + await mailbox.deleteThreadIfEmpty({ + ownerId: userId, + threadId: sharedThread.id, + deletedAt: '2026-07-02T10:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) expect(await mailbox.getThread({ threadId: sharedThread.id })).not.toBeNull() }) @@ -328,30 +395,33 @@ test('Mailbox deleteDeliveryEvent: stale, deleted, and missing outcomes', async }), }) - const staleDelete = await mailbox.deleteDeliveryEvent({ - ownerId: userId, - eventId: 'evt-del', - deletedAt: '2026-07-02T09:00:00.000Z', - }) - expect(staleDelete).toEqual({ deleted: false, stale: true }) + expect( + await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T09:00:00.000Z', + }), + ).toEqual({ status: 'stale' }) expect( await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), ).toHaveLength(1) - const deleted = await mailbox.deleteDeliveryEvent({ - ownerId: userId, - eventId: 'evt-del', - deletedAt: '2026-07-02T10:00:00.000Z', - }) - expect(deleted).toEqual({ deleted: true, stale: false }) + expect( + await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T10:00:00.000Z', + }), + ).toEqual({ status: 'deleted' }) expect( await mailbox.listDeliveryEvents({ messageId: message.id, limit: 5 }), ).toHaveLength(0) - const again = await mailbox.deleteDeliveryEvent({ - ownerId: userId, - eventId: 'evt-del', - deletedAt: '2026-07-02T11:00:00.000Z', - }) - expect(again).toEqual({ deleted: false, stale: false }) + expect( + await mailbox.deleteDeliveryEvent({ + ownerId: userId, + eventId: 'evt-del', + deletedAt: '2026-07-02T11:00:00.000Z', + }), + ).toEqual({ status: 'missing' }) }) diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 1cc170a450..2b70d35ecc 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -59,37 +59,53 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = expect(writeDataPoint).toHaveBeenCalledTimes(6) expect(writeDataPoint).toHaveBeenNthCalledWith(1, { indexes: ['user-1'], - blobs: ['mirror_message', 'mirrored', '2026-07-30T10:00:00.000Z'], - doubles: [0, 1], + blobs: [ + 'mailbox_mirror:mirror_message', + 'mirrored', + '2026-07-30T10:00:00.000Z', + ], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(2, { indexes: ['user-1'], - blobs: ['touch_thread', 'stale', '2026-07-30T10:05:00.000Z'], - doubles: [0, 1], + blobs: ['mailbox_mirror:touch_thread', 'stale', '2026-07-30T10:05:00.000Z'], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(3, { indexes: ['user-1'], blobs: [ - 'set_message_classification', + 'mailbox_mirror:set_message_classification', 'skipped', '2026-07-30T10:06:00.000Z', ], - doubles: [0, 1], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(4, { indexes: ['user-1'], - blobs: ['delete_message_metadata', 'error', '2026-07-30T10:07:00.000Z'], - doubles: [0, 1], + blobs: [ + 'mailbox_mirror:delete_message_metadata', + 'error', + '2026-07-30T10:07:00.000Z', + ], + doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(5, { indexes: ['user-2'], - blobs: ['compare_messages', 'mismatch', '2026-07-30T11:00:00.000Z'], - doubles: [2, 1], + blobs: [ + 'mailbox_parity:compare_messages', + 'mismatch', + '2026-07-30T11:00:00.000Z', + ], + doubles: [1, 2], }) expect(writeDataPoint).toHaveBeenNthCalledWith(6, { indexes: ['user-3'], - blobs: ['compare_threads', 'match', '2026-07-30T12:00:00.000Z'], - doubles: [0, 1], + blobs: [ + 'mailbox_parity:compare_threads', + 'match', + '2026-07-30T12:00:00.000Z', + ], + doubles: [1, 0], }) }) diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts index e3dcd74229..66f394eee9 100644 --- a/packages/worker/src/email/mailbox-parity-events.ts +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -12,6 +12,7 @@ export type MailboxMirrorOperation = | 'set_message_classification' | 'delete_message_metadata' | 'delete_delivery_event' + | 'delete_thread_if_empty' export type MailboxMirrorOutcome = 'mirrored' | 'stale' | 'skipped' | 'error' @@ -52,15 +53,23 @@ function resolveCountDelta(event: MailboxParityEvent): number { return 0 } +function mailboxParityEventType(event: MailboxParityEvent): string { + const prefix = + event.category === 'mirror' ? 'mailbox_mirror' : 'mailbox_parity' + return `${prefix}:${event.operation}` +} + /** * Additive Analytics Engine schema (same EMAIL_EVENTS dataset as reporting): * * - index1: stable user id (per-user isolation) - * - blob1: mirror or parity operation + * - blob1: namespaced event type (`mailbox_mirror:` or + * `mailbox_parity:`) so unfiltered consumers can distinguish these + * from `email_send` / `email_receive` / `email_delivery` * - blob2: operation outcome * - blob3: source event timestamp (ISO 8601) - * - double1: D1-minus-DO count delta (0 when absent or not applicable) - * - double2: event weight (always 1) + * - double1: event weight (always 1, matching reporting) + * - double2: D1-minus-DO count delta (0 when absent or not applicable) * * `system:email` is excluded — operator mail stays in D1 only. */ @@ -75,11 +84,11 @@ export function recordMailboxParityEvent( env.EMAIL_EVENTS.writeDataPoint({ indexes: [event.userId], blobs: [ - event.operation, + mailboxParityEventType(event), event.outcome, event.timestamp ?? new Date().toISOString(), ], - doubles: [resolveCountDelta(event), 1], + doubles: [1, resolveCountDelta(event)], }) } catch (error) { console.warn('mailbox-parity-event-write-failed', error) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts new file mode 100644 index 0000000000..fb55632566 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts @@ -0,0 +1,120 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { + getEmailDeliveryEventMirrorProjection, + getMailboxDeliveryEventMirrorInput, +} from './mailbox-snapshot-repo.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +async function createEmailDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +test('mailbox-snapshot-repo loads complete delivery-event projection and ready Mailbox input', async () => { + const { db } = await createEmailDb() + const detail = { + state: 'received', + fingerprint: 'fp-repo', + storageLease: 'lease-repo', + subscriptionEffectState: 'pending', + usageStartedAt: '2026-07-02T09:59:00.000Z', + // Stale JSON copies of promoted columns must not win after conversion. + usageEffectRecordedAt: '2026-01-01T00:00:00.000Z', + usageMonth: 'json-month', + usageBytes: 1, + usageDurationMs: 1, + } + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, usage_effect_recorded_at, usage_month, + usage_bytes, usage_duration_ms, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'evt-1', + 'msg-1', + 'user-aaa', + 'inbox-1', + 'received', + 'cloudflare-email-routing', + null, + 'provider-evt-1', + JSON.stringify(detail), + 1, + '2026-07-02T10:01:00.000Z', + '2026-07', + 2048, + 120, + '2026-07-02T10:00:00.000Z', + ) + .run() + + const projection = await getEmailDeliveryEventMirrorProjection({ + db, + ownerId: 'user-aaa', + eventId: 'evt-1', + }) + expect(projection).toEqual({ + id: 'evt-1', + messageId: 'msg-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + eventType: 'received', + provider: 'cloudflare-email-routing', + providerMessageId: null, + providerEventId: 'provider-evt-1', + detailJson: JSON.stringify(detail), + createdAt: '2026-07-02T10:00:00.000Z', + needsEffectReconcile: true, + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + }) + + expect( + await getEmailDeliveryEventMirrorProjection({ + db, + ownerId: 'other-user', + eventId: 'evt-1', + }), + ).toBeNull() + expect( + await getEmailDeliveryEventMirrorProjection({ + db, + ownerId: 'user-aaa', + eventId: 'missing', + }), + ).toBeNull() + + const mailboxInput = await getMailboxDeliveryEventMirrorInput({ + db, + ownerId: 'user-aaa', + eventId: 'evt-1', + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + expect(mailboxInput).toEqual( + expect.objectContaining({ + id: 'evt-1', + needsEffectReconcile: true, + state: 'received', + fingerprint: 'fp-repo', + storageLease: 'lease-repo', + subscriptionEffectState: 'pending', + usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', + usageMonth: '2026-07', + usageBytes: 2048, + usageDurationMs: 120, + usageStartedAt: '2026-07-02T09:59:00.000Z', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + ) +}) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts new file mode 100644 index 0000000000..30df8d8ea3 --- /dev/null +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -0,0 +1,126 @@ +import { + toMailboxDeliveryEventInput, + type EmailDeliveryEventMirrorProjection, +} from './mailbox-snapshots.ts' +import { type EmailDeliveryEventType } from './types.ts' +import { type MailboxDeliveryEventInput } from './mailbox-types.ts' + +/** + * D1 loaders for complete Mailbox mirror projections. + * + * Delivery-event dual-write callers load one cohesive projection here instead + * of stitching promoted columns into the converter by hand. Detail JSON + * decoding stays centralized in `mailbox-snapshots.ts`. + * + * Phase-2 high-risk lifecycle mutations must pass the same canonical + * `sourceMutationAt` used for the D1 mutation as the Mailbox `updatedAt`. + * Inserts use `created_at`. + */ + +type DeliveryEventMirrorRow = { + id: string + message_id: string | null + user_id: string | null + inbox_id: string | null + event_type: string + provider: string | null + provider_message_id: string | null + provider_event_id: string | null + detail_json: string + needs_effect_reconcile: number + usage_effect_recorded_at: string | null + usage_month: string | null + usage_bytes: number | null + usage_duration_ms: number | null + created_at: string +} + +function mapDeliveryEventMirrorRow( + row: DeliveryEventMirrorRow, +): EmailDeliveryEventMirrorProjection { + return { + id: String(row.id), + messageId: row.message_id == null ? null : String(row.message_id), + userId: row.user_id == null ? null : String(row.user_id), + inboxId: row.inbox_id == null ? null : String(row.inbox_id), + eventType: String(row.event_type) as EmailDeliveryEventType, + provider: row.provider == null ? null : String(row.provider), + providerMessageId: + row.provider_message_id == null ? null : String(row.provider_message_id), + providerEventId: + row.provider_event_id == null ? null : String(row.provider_event_id), + detailJson: String(row.detail_json ?? '{}'), + createdAt: String(row.created_at), + needsEffectReconcile: Number(row.needs_effect_reconcile ?? 0) === 1, + usageEffectRecordedAt: + row.usage_effect_recorded_at == null + ? null + : String(row.usage_effect_recorded_at), + usageMonth: row.usage_month == null ? null : String(row.usage_month), + usageBytes: row.usage_bytes == null ? null : Number(row.usage_bytes) || 0, + usageDurationMs: + row.usage_duration_ms == null ? null : Number(row.usage_duration_ms) || 0, + } +} + +/** + * Load one complete delivery-event mirror projection scoped by owner + id. + * Returns null when the row is absent or owned by a different user. + */ +export async function getEmailDeliveryEventMirrorProjection(input: { + db: D1Database + ownerId: string + eventId: string +}): Promise { + const row = await input.db + .prepare( + `SELECT + id, + message_id, + user_id, + inbox_id, + event_type, + provider, + provider_message_id, + provider_event_id, + detail_json, + needs_effect_reconcile, + usage_effect_recorded_at, + usage_month, + usage_bytes, + usage_duration_ms, + created_at + FROM email_delivery_events + WHERE id = ? + AND user_id = ? + LIMIT 1`, + ) + .bind(input.eventId, input.ownerId) + .first() + return row ? mapDeliveryEventMirrorRow(row) : null +} + +/** + * Load and convert one ready Mailbox delivery-event snapshot for dual-write. + * + * @param sourceMutationAt - Canonical mirror `updatedAt`. D1 delivery events + * lack `updated_at`. Phase-2 high-risk lifecycle mutations must pass the + * same timestamp used for the D1 mutation; inserts use `created_at`. + */ +export async function getMailboxDeliveryEventMirrorInput(input: { + db: D1Database + ownerId: string + eventId: string + sourceMutationAt: string +}): Promise { + const projection = await getEmailDeliveryEventMirrorProjection({ + db: input.db, + ownerId: input.ownerId, + eventId: input.eventId, + }) + if (!projection) return null + return toMailboxDeliveryEventInput({ + projection, + sourceMutationAt: input.sourceMutationAt, + }) +} diff --git a/packages/worker/src/email/mailbox-snapshots.node.test.ts b/packages/worker/src/email/mailbox-snapshots.node.test.ts index bd26855c74..7b6ba6a95f 100644 --- a/packages/worker/src/email/mailbox-snapshots.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshots.node.test.ts @@ -1,7 +1,6 @@ import { expect, test } from 'vitest' import { type EmailAttachmentRecord, - type EmailDeliveryEventRecord, type EmailMessageRecord, type EmailThreadRecord, } from './types.ts' @@ -10,7 +9,7 @@ import { toMailboxDeliveryEventInput, toMailboxMessageInput, toMailboxThreadInput, - type EmailDeliveryEventMirrorSnapshot, + type EmailDeliveryEventMirrorProjection, } from './mailbox-snapshots.ts' test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted columns', () => { @@ -129,8 +128,6 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted createdAt: '2026-07-01T12:00:00.000Z', }) - // detail_json still owns inbound/effect lease fields, but also may contain - // stale copies of promoted columns — converter must prefer D1 columns. const detail = { state: 'received', fingerprint: 'fp-1', @@ -161,7 +158,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted subscriptionEffectLastError: null, recipient: 'owner@example.com', } - const event: EmailDeliveryEventRecord = { + const projection: EmailDeliveryEventMirrorProjection = { id: 'evt-1', messageId: 'msg-1', userId: 'user-1', @@ -172,17 +169,18 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted providerEventId: 'provider-evt-1', detailJson: JSON.stringify(detail), createdAt: '2026-07-02T10:00:00.000Z', - } - const snapshot: EmailDeliveryEventMirrorSnapshot = { - event, - updatedAt: '2026-07-02T10:00:01.000Z', needsEffectReconcile: true, usageEffectRecordedAt: '2026-07-02T10:01:00.000Z', usageMonth: '2026-07', usageBytes: 2048, usageDurationMs: 120, } - expect(toMailboxDeliveryEventInput(snapshot)).toEqual({ + expect( + toMailboxDeliveryEventInput({ + projection, + sourceMutationAt: '2026-07-02T10:00:01.000Z', + }), + ).toEqual({ id: 'evt-1', messageId: 'msg-1', inboxId: 'inbox-1', @@ -190,7 +188,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted provider: 'kody', providerMessageId: null, providerEventId: 'provider-evt-1', - detailJson: event.detailJson, + detailJson: projection.detailJson, needsEffectReconcile: true, state: 'received', fingerprint: 'fp-1', @@ -225,7 +223,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted }) test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or enums', () => { - const baseEvent: EmailDeliveryEventRecord = { + const baseProjection: EmailDeliveryEventMirrorProjection = { id: 'evt-bad', messageId: null, userId: 'user-1', @@ -236,59 +234,56 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or providerEventId: null, detailJson: '{}', createdAt: '2026-07-02T10:00:00.000Z', - } - const baseSnapshot = { - updatedAt: '2026-07-02T10:00:00.000Z', needsEffectReconcile: false, usageEffectRecordedAt: null, usageMonth: null, usageBytes: null, usageDurationMs: null, - } satisfies Omit + } expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { ...baseEvent, detailJson: '{not-json' }, + projection: { ...baseProjection, detailJson: '{not-json' }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson is not valid JSON/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { ...baseEvent, detailJson: '[]' }, + projection: { ...baseProjection, detailJson: '[]' }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson must be a JSON object/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { - ...baseEvent, + projection: { + ...baseProjection, detailJson: JSON.stringify({ state: 'not-a-state' }), }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.state is invalid/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { - ...baseEvent, + projection: { + ...baseProjection, detailJson: JSON.stringify({ subscriptionEffectState: 'bogus', }), }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.subscriptionEffectState is invalid/) expect(() => toMailboxDeliveryEventInput({ - ...baseSnapshot, - event: { - ...baseEvent, + projection: { + ...baseProjection, detailJson: JSON.stringify({ expectedAttachmentCount: 'lots' }), }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.expectedAttachmentCount must be a finite number/) diff --git a/packages/worker/src/email/mailbox-snapshots.ts b/packages/worker/src/email/mailbox-snapshots.ts index f2bd487315..839001d35f 100644 --- a/packages/worker/src/email/mailbox-snapshots.ts +++ b/packages/worker/src/email/mailbox-snapshots.ts @@ -1,6 +1,6 @@ import { type EmailAttachmentRecord, - type EmailDeliveryEventRecord, + type EmailDeliveryEventType, type EmailMessageRecord, type EmailThreadRecord, } from './types.ts' @@ -23,38 +23,42 @@ import { * Pure D1 → Mailbox complete-snapshot converters. * * Normalize nulls to Mailbox SQLite defaults (empty strings, `[]`, `{}`, `0`, - * `application/octet-stream`, `kody`). Delivery-event conversion takes an - * additive {@link EmailDeliveryEventMirrorSnapshot}: D1-promoted columns are - * authoritative; `detail_json` supplies only fields that remain JSON-owned. - * Invalid persisted JSON/enums fail clearly rather than fabricating state. + * `application/octet-stream`, `kody`). Delivery-event conversion consumes a + * complete {@link EmailDeliveryEventMirrorProjection} (base + promoted D1 + * columns) plus an explicit `sourceMutationAt`. Invalid persisted JSON/enums + * fail clearly rather than fabricating state. + * + * Load projections via `mailbox-snapshot-repo.ts` — callers must not assemble + * promoted columns field-by-field. */ /** - * Additive delivery-event mirror input. D1 `EmailDeliveryEventRecord` does not - * yet expose promoted columns or `updated_at`; callers pass those explicitly. + * Complete D1 delivery-event mirror projection: base row fields plus + * authoritative promoted columns. `detail_json` still owns inbound/effect + * lease fields that have not been promoted. * - * Authoritative D1 columns (do not read these from `detail_json`): + * Authoritative D1 columns (never read from `detail_json`): * - `needs_effect_reconcile` * - `usage_effect_recorded_at` * - `usage_month` * - `usage_bytes` * - `usage_duration_ms` - * - * Still JSON-owned in `detail_json` (inbound/effect lease fields, etc.). */ -export type EmailDeliveryEventMirrorSnapshot = { - event: EmailDeliveryEventRecord - /** D1 delivery events lack `updated_at`; required for stale rejection. */ - updatedAt: string - /** D1 `needs_effect_reconcile` column. */ +export type EmailDeliveryEventMirrorProjection = { + id: string + messageId: string | null + userId: string | null + inboxId: string | null + eventType: EmailDeliveryEventType + provider: string | null + providerMessageId: string | null + providerEventId: string | null + detailJson: string + createdAt: string needsEffectReconcile: boolean - /** D1 `usage_effect_recorded_at` column. */ usageEffectRecordedAt: string | null - /** D1 `usage_month` column. */ usageMonth: string | null - /** D1 `usage_bytes` column. */ usageBytes: number | null - /** D1 `usage_duration_ms` column. */ usageDurationMs: number | null } @@ -190,73 +194,41 @@ export function toMailboxAttachmentInput( } /** - * Convert a D1 delivery-event mirror snapshot into a full Mailbox input. - * Promoted D1 columns win; remaining inbound/effect fields come from JSON. + * Convert a complete D1 delivery-event projection into a Mailbox input. + * Promoted columns win; remaining inbound/effect fields come from JSON. + * + * `sourceMutationAt` becomes Mailbox `updatedAt`. D1 delivery events lack + * `updated_at`: phase-2 high-risk lifecycle mutations must pass the same + * canonical timestamp used for the D1 mutation; inserts use `created_at`. */ -export function toMailboxDeliveryEventInput( - snapshot: EmailDeliveryEventMirrorSnapshot, -): MailboxDeliveryEventInput { - const { - event, - updatedAt, - needsEffectReconcile, - usageEffectRecordedAt, - usageMonth, - usageBytes, - usageDurationMs, - } = snapshot - if (typeof needsEffectReconcile !== 'boolean') { - throw new Error( - `Mailbox snapshot needsEffectReconcile must be a boolean; got ${JSON.stringify(needsEffectReconcile)}.`, - ) - } - if (typeof updatedAt !== 'string' || updatedAt.length === 0) { - throw new Error( - `Mailbox snapshot updatedAt must be a non-empty string; got ${JSON.stringify(updatedAt)}.`, - ) - } - if ( - usageEffectRecordedAt != null && - typeof usageEffectRecordedAt !== 'string' - ) { +export function toMailboxDeliveryEventInput(input: { + projection: EmailDeliveryEventMirrorProjection + sourceMutationAt: string +}): MailboxDeliveryEventInput { + const { projection, sourceMutationAt } = input + if (typeof sourceMutationAt !== 'string' || sourceMutationAt.length === 0) { throw new Error( - `Mailbox snapshot usageEffectRecordedAt must be a string or null; got ${JSON.stringify(usageEffectRecordedAt)}.`, + `Mailbox snapshot sourceMutationAt must be a non-empty string; got ${JSON.stringify(sourceMutationAt)}.`, ) } - if (usageMonth != null && typeof usageMonth !== 'string') { - throw new Error( - `Mailbox snapshot usageMonth must be a string or null; got ${JSON.stringify(usageMonth)}.`, - ) - } - if ( - usageBytes != null && - (typeof usageBytes !== 'number' || !Number.isFinite(usageBytes)) - ) { - throw new Error( - `Mailbox snapshot usageBytes must be a finite number or null; got ${JSON.stringify(usageBytes)}.`, - ) - } - if ( - usageDurationMs != null && - (typeof usageDurationMs !== 'number' || !Number.isFinite(usageDurationMs)) - ) { + if (typeof projection.needsEffectReconcile !== 'boolean') { throw new Error( - `Mailbox snapshot usageDurationMs must be a finite number or null; got ${JSON.stringify(usageDurationMs)}.`, + `Mailbox snapshot needsEffectReconcile must be a boolean; got ${JSON.stringify(projection.needsEffectReconcile)}.`, ) } - const detail = parseDetailRecord(event.detailJson) + const detail = parseDetailRecord(projection.detailJson) return { - id: event.id, - messageId: event.messageId, - inboxId: event.inboxId, - eventType: assertMailboxDeliveryEventType(event.eventType), - provider: event.provider ?? 'kody', - providerMessageId: event.providerMessageId, - providerEventId: event.providerEventId, - detailJson: event.detailJson, - needsEffectReconcile, + id: projection.id, + messageId: projection.messageId, + inboxId: projection.inboxId, + eventType: assertMailboxDeliveryEventType(projection.eventType), + provider: projection.provider ?? 'kody', + providerMessageId: projection.providerMessageId, + providerEventId: projection.providerEventId, + detailJson: projection.detailJson, + needsEffectReconcile: projection.needsEffectReconcile, state: requireOptionalEnum( mailboxInboundDeliveryStateValues, detail['state'], @@ -303,7 +275,7 @@ export function toMailboxDeliveryEventInput( 'detail.dedupeExpiresAt', ), // Authoritative D1 columns — ignore any JSON copies of these keys. - usageEffectRecordedAt, + usageEffectRecordedAt: projection.usageEffectRecordedAt, usageEffectSuppressedAt: requireOptionalString( detail['usageEffectSuppressedAt'], 'detail.usageEffectSuppressedAt', @@ -312,9 +284,9 @@ export function toMailboxDeliveryEventInput( detail['usageStartedAt'], 'detail.usageStartedAt', ), - usageMonth, - usageBytes, - usageDurationMs, + usageMonth: projection.usageMonth, + usageBytes: projection.usageBytes, + usageDurationMs: projection.usageDurationMs, usageEffectRetryAt: requireOptionalString( detail['usageEffectRetryAt'], 'detail.usageEffectRetryAt', @@ -356,7 +328,7 @@ export function toMailboxDeliveryEventInput( detail['subscriptionEffectLastError'], 'detail.subscriptionEffectLastError', ), - createdAt: event.createdAt, - updatedAt, + createdAt: projection.createdAt, + updatedAt: sourceMutationAt, } } diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 3d9ea8b629..75b38f9927 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -11,13 +11,10 @@ import { encodeMailboxListCursor, mailboxNowIso, normalizeMailboxPageSize, - type MailboxAcceptedResult, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, type MailboxCountResult, - type MailboxDeleteDeliveryEventResult, - type MailboxDeleteMessageMetadataResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, @@ -25,19 +22,9 @@ import { type MailboxMessageInput, type MailboxMessageRecord, type MailboxSearchMessagesInput, - type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, - type MailboxTouchThreadInput, - type MailboxUpdateMessageDeliveryInput, } from './mailbox-types.ts' -import { - deleteMailboxDeliveryEvent, - deleteMailboxMessageMetadata, - setMailboxMessageClassification, - touchMailboxThread, - updateMailboxMessageDelivery, -} from './mailbox-mutations.ts' import { type EmailDeliveryEventType, type EmailDeliveryStatus, @@ -454,38 +441,6 @@ export class MailboxStore { return deliveryEventOwnsMessage(this.sql, eventId, messageId) } - touchThread( - input: Omit, - ): MailboxAcceptedResult { - return touchMailboxThread(this.sql, input) - } - - updateMessageDelivery( - input: Omit, - ): MailboxAcceptedResult { - return updateMailboxMessageDelivery(this.sql, input) - } - - setMessageClassification( - input: Omit, - ): MailboxAcceptedResult { - return setMailboxMessageClassification(this.sql, input) - } - - deleteMessageMetadata(input: { - messageId: string - deletedAt: string - }): MailboxDeleteMessageMetadataResult { - return deleteMailboxMessageMetadata(this.sql, input) - } - - deleteDeliveryEvent(input: { - eventId: string - deletedAt: string - }): MailboxDeleteDeliveryEventResult { - return deleteMailboxDeliveryEvent(this.sql, input) - } - getThread(threadId: string): MailboxThreadRecord | null { const row = this.sql .exec>( diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 6178ce59b6..5855ebce86 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -382,8 +382,9 @@ export type MailboxSetMessageClassificationInput = { } /** - * Metadata-only message delete. Never touches R2. Stale when the row's - * `updated_at` is newer than `deletedAt`. Idempotent when already absent. + * Metadata-only message delete. Never touches R2. Nulls matching delivery + * event `message_id` first; does not delete empty threads (use + * `deleteThreadIfEmpty`). Stale when `updated_at` is newer than `deletedAt`. */ export type MailboxDeleteMessageMetadataInput = { ownerId: string @@ -401,35 +402,37 @@ export type MailboxDeleteDeliveryEventInput = { deletedAt: string } -export type MailboxAcceptedResult = { - accepted: boolean +/** + * Deferred empty-thread cleanup (D1 `deleteEmptyEmailThreads` parity). + * Stale-safe by `thread.updated_at`; no-op when messages remain. + */ +export type MailboxDeleteThreadIfEmptyInput = { + ownerId: string + threadId: string + deletedAt: string } /** - * Message metadata delete outcome. - * - `deleted: true` — row removed; includes `orphanThreadDeleted` - * - `deleted: false, stale: false` — missing (idempotent success) - * - `deleted: false, stale: true` — newer row retained - * - * Best-effort callers map missing (`!deleted && !stale`) to success. + * Partial touch/update/classify outcome. + * - `accepted` — mutation applied + * - `missing` — target row absent (idempotent success for best-effort callers) + * - `stale` — newer `updated_at` retained */ -export type MailboxDeleteMessageMetadataResult = - | { deleted: true; stale: false; orphanThreadDeleted: boolean } - | { deleted: false; stale: true } - | { deleted: false; stale: false } +export type MailboxPartialMutationResult = + | { status: 'accepted' } + | { status: 'missing' } + | { status: 'stale' } /** - * Delivery-event delete outcome. - * - `deleted: true` — row removed - * - `deleted: false, stale: false` — missing (idempotent success) - * - `deleted: false, stale: true` — newer row retained - * - * Best-effort callers map missing (`!deleted && !stale`) to success. + * Metadata delete outcome (message, delivery event, or empty thread). + * - `deleted` — row removed + * - `missing` — already absent or nothing to delete (idempotent success) + * - `stale` — newer `updated_at` retained */ -export type MailboxDeleteDeliveryEventResult = - | { deleted: true; stale: false } - | { deleted: false; stale: true } - | { deleted: false; stale: false } +export type MailboxDeleteResult = + | { status: 'deleted' } + | { status: 'missing' } + | { status: 'stale' } export type MailboxRpc = { mirrorMessage: (input: { @@ -462,19 +465,22 @@ export type MailboxRpc = { }> touchThread: ( input: MailboxTouchThreadInput, - ) => Promise + ) => Promise updateMessageDelivery: ( input: MailboxUpdateMessageDeliveryInput, - ) => Promise + ) => Promise setMessageClassification: ( input: MailboxSetMessageClassificationInput, - ) => Promise + ) => Promise deleteMessageMetadata: ( input: MailboxDeleteMessageMetadataInput, - ) => Promise + ) => Promise deleteDeliveryEvent: ( input: MailboxDeleteDeliveryEventInput, - ) => Promise + ) => Promise + deleteThreadIfEmpty: ( + input: MailboxDeleteThreadIfEmptyInput, + ) => Promise getThread: (input: { threadId: string }) => Promise From ca0e6666e72521159b5ad6c4ebf671703af41b32 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 09:55:22 +0000 Subject: [PATCH 06/62] fix(email): harden Mailbox snapshot validation Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 2 +- .../src/email/mailbox-mirror.node.test.ts | 11 +++ .../worker/src/email/mailbox-mutations.ts | 6 +- .../email/mailbox-parity-events.node.test.ts | 13 ++++ .../worker/src/email/mailbox-snapshot-repo.ts | 4 +- .../src/email/mailbox-snapshots.node.test.ts | 70 ++++++++++++++++++- .../worker/src/email/mailbox-snapshots.ts | 34 +++++++-- 7 files changed, 127 insertions(+), 13 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 402bf79a48..0acb5fb677 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -763,7 +763,7 @@ surface that later live-path wiring will call. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO completeness against D1. D1 stays authoritative for reads. Phase 2a (current - PR) adds `email-owner.ts`, `mailbox-snapshots.ts`, + PR) adds `email-owner.ts`, `mailbox-types.ts`, `mailbox-snapshots.ts`, `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index b7bc8344e0..0bc60502e5 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -349,4 +349,15 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep message: baseMessage(), }), ).toEqual({ status: 'skipped', reason: 'mailbox-unconfigured' }) + + expect( + await mirrorMailboxTouchThread({ + env, + ownerId: '', + threadId: 'thread-1', + lastMessageAt: '2026-07-01T13:00:00.000Z', + updatedAt: '2026-07-01T13:00:00.000Z', + }), + ).toEqual({ status: 'skipped', reason: 'missing-owner' }) + expect(touchThread).toHaveBeenCalledTimes(1) }) diff --git a/packages/worker/src/email/mailbox-mutations.ts b/packages/worker/src/email/mailbox-mutations.ts index 039985d632..69bcd19e79 100644 --- a/packages/worker/src/email/mailbox-mutations.ts +++ b/packages/worker/src/email/mailbox-mutations.ts @@ -4,6 +4,8 @@ import { assertMailboxNonEmptyString, assertMailboxProcessingStatus, assertOptionalMailboxCanonicalIsoTimestamp, + type MailboxDeleteDeliveryEventInput, + type MailboxDeleteMessageMetadataInput, type MailboxDeleteResult, type MailboxDeleteThreadIfEmptyInput, type MailboxPartialMutationResult, @@ -141,7 +143,7 @@ export function setMailboxMessageClassification( */ export function deleteMailboxMessageMetadata( sql: SqlStorage, - input: { messageId: string; deletedAt: string }, + input: Omit, ): MailboxDeleteResult { const messageId = assertMailboxNonEmptyString(input.messageId, 'messageId') const deletedAt = assertMailboxCanonicalIsoTimestamp( @@ -182,7 +184,7 @@ export function deleteMailboxMessageMetadata( */ export function deleteMailboxDeliveryEvent( sql: SqlStorage, - input: { eventId: string; deletedAt: string }, + input: Omit, ): MailboxDeleteResult { const eventId = assertMailboxNonEmptyString(input.eventId, 'eventId') const deletedAt = assertMailboxCanonicalIsoTimestamp( diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 2b70d35ecc..93c755631c 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -128,6 +128,19 @@ test('mailbox parity skips system email and never throws when the binding is abs ) expect(writeDataPoint).not.toHaveBeenCalled() + recordMailboxParityEvent( + { + EMAIL_EVENTS: { writeDataPoint } as unknown as AnalyticsEngineDataset, + }, + { + userId: '', + category: 'mirror', + operation: 'mirror_message', + outcome: 'mirrored', + }, + ) + expect(writeDataPoint).not.toHaveBeenCalled() + expect(() => recordMailboxParityEvent( {}, diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index 30df8d8ea3..5fafe8ee09 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -57,9 +57,9 @@ function mapDeliveryEventMirrorRow( ? null : String(row.usage_effect_recorded_at), usageMonth: row.usage_month == null ? null : String(row.usage_month), - usageBytes: row.usage_bytes == null ? null : Number(row.usage_bytes) || 0, + usageBytes: row.usage_bytes == null ? null : Number(row.usage_bytes), usageDurationMs: - row.usage_duration_ms == null ? null : Number(row.usage_duration_ms) || 0, + row.usage_duration_ms == null ? null : Number(row.usage_duration_ms), } } diff --git a/packages/worker/src/email/mailbox-snapshots.node.test.ts b/packages/worker/src/email/mailbox-snapshots.node.test.ts index 7b6ba6a95f..b944ff972b 100644 --- a/packages/worker/src/email/mailbox-snapshots.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshots.node.test.ts @@ -223,6 +223,7 @@ test('D1→Mailbox snapshot converters normalize defaults and prefer D1 promoted }) test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or enums', () => { + const sentinelSecret = 'SENTINEL_SECRET_DO_NOT_LEAK_9f3a2b1c' const baseProjection: EmailDeliveryEventMirrorProjection = { id: 'evt-bad', messageId: null, @@ -243,10 +244,27 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or expect(() => toMailboxDeliveryEventInput({ - projection: { ...baseProjection, detailJson: '{not-json' }, + projection: baseProjection, + sourceMutationAt: '', + }), + ).toThrow(/sourceMutationAt must be a non-empty string/) + + const invalidJsonDetail = `{not-json,"apiKey":"${sentinelSecret}"` + expect(() => + toMailboxDeliveryEventInput({ + projection: { ...baseProjection, detailJson: invalidJsonDetail }, sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson is not valid JSON/) + try { + toMailboxDeliveryEventInput({ + projection: { ...baseProjection, detailJson: invalidJsonDetail }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + expect(String(error)).toMatch(/payload is \d+ bytes/) + } expect(() => toMailboxDeliveryEventInput({ @@ -254,6 +272,18 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detailJson must be a JSON object/) + try { + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + detailJson: JSON.stringify([sentinelSecret]), + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + expect(String(error)).toMatch(/array \(1 items\)/) + } expect(() => toMailboxDeliveryEventInput({ @@ -264,6 +294,20 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.state is invalid/) + try { + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + detailJson: JSON.stringify({ + state: 'not-a-state', + apiKey: sentinelSecret, + }), + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + } expect(() => toMailboxDeliveryEventInput({ @@ -286,6 +330,30 @@ test('D1→Mailbox delivery snapshot conversion fails clearly on invalid JSON or sourceMutationAt: '2026-07-02T10:00:00.000Z', }), ).toThrow(/detail\.expectedAttachmentCount must be a finite number/) + try { + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + detailJson: JSON.stringify({ + expectedAttachmentCount: sentinelSecret, + }), + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }) + } catch (error) { + expect(String(error)).not.toContain(sentinelSecret) + expect(String(error)).toMatch(/string \(\d+ chars\)/) + } + + expect(() => + toMailboxDeliveryEventInput({ + projection: { + ...baseProjection, + usageBytes: Number.NaN, + }, + sourceMutationAt: '2026-07-02T10:00:00.000Z', + }), + ).toThrow(/usageBytes must be a finite number/) expect(() => toMailboxAttachmentInput({ diff --git a/packages/worker/src/email/mailbox-snapshots.ts b/packages/worker/src/email/mailbox-snapshots.ts index 839001d35f..20d2c5a6c3 100644 --- a/packages/worker/src/email/mailbox-snapshots.ts +++ b/packages/worker/src/email/mailbox-snapshots.ts @@ -62,11 +62,28 @@ export type EmailDeliveryEventMirrorProjection = { usageDurationMs: number | null } +function describeSnapshotValue(value: unknown): string { + if (value === null) return 'null' + if (value === undefined) return 'undefined' + if (typeof value === 'string') { + return `string (${value.length} chars)` + } + if (typeof value === 'number') { + return Number.isFinite(value) ? 'number' : 'number (non-finite)' + } + if (typeof value === 'boolean') return 'boolean' + if (Array.isArray(value)) return `array (${value.length} items)` + if (typeof value === 'object') { + return `object (${Object.keys(value as Record).length} keys)` + } + return typeof value +} + function requireOptionalString(value: unknown, label: string): string | null { if (value == null) return null if (typeof value !== 'string') { throw new Error( - `Mailbox snapshot ${label} must be a string or null; got ${JSON.stringify(value)}.`, + `Mailbox snapshot ${label} must be a string or null; got ${describeSnapshotValue(value)}.`, ) } return value @@ -76,7 +93,7 @@ function requireOptionalNumber(value: unknown, label: string): number | null { if (value == null) return null if (typeof value !== 'number' || !Number.isFinite(value)) { throw new Error( - `Mailbox snapshot ${label} must be a finite number or null; got ${JSON.stringify(value)}.`, + `Mailbox snapshot ${label} must be a finite number or null; got ${describeSnapshotValue(value)}.`, ) } return value @@ -93,7 +110,7 @@ function requireOptionalEnum( !(values as ReadonlyArray).includes(value) ) { throw new Error( - `Mailbox snapshot ${label} is invalid; got ${JSON.stringify(value)}.`, + `Mailbox snapshot ${label} is invalid; got ${describeSnapshotValue(value)}.`, ) } return value as T @@ -105,12 +122,12 @@ function parseDetailRecord(detailJson: string): Record { parsed = JSON.parse(detailJson) as unknown } catch { throw new Error( - `Mailbox snapshot detailJson is not valid JSON; got ${JSON.stringify(detailJson)}.`, + `Mailbox snapshot detailJson is not valid JSON; payload is ${detailJson.length} bytes.`, ) } if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { throw new Error( - `Mailbox snapshot detailJson must be a JSON object; got ${JSON.stringify(parsed)}.`, + `Mailbox snapshot detailJson must be a JSON object; got ${describeSnapshotValue(parsed)}.`, ) } return parsed as Record @@ -285,8 +302,11 @@ export function toMailboxDeliveryEventInput(input: { 'detail.usageStartedAt', ), usageMonth: projection.usageMonth, - usageBytes: projection.usageBytes, - usageDurationMs: projection.usageDurationMs, + usageBytes: requireOptionalNumber(projection.usageBytes, 'usageBytes'), + usageDurationMs: requireOptionalNumber( + projection.usageDurationMs, + 'usageDurationMs', + ), usageEffectRetryAt: requireOptionalString( detail['usageEffectRetryAt'], 'detail.usageEffectRetryAt', From ef268fe750d9e40c5bf7a5852d156a62a7e24a61 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 10:26:09 +0000 Subject: [PATCH 07/62] feat(email): dual-write non-inbound Mailbox state Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 146 ++++-- .../app/handlers/account-email.node.test.ts | 52 ++ .../worker/src/app/handlers/account-email.ts | 8 + packages/worker/src/email/delivery-events.ts | 43 +- .../src/email/delivery-events.workers.test.ts | 140 ++++++ .../src/email/delivery-queue.node.test.ts | 156 +++++- packages/worker/src/email/delivery-queue.ts | 19 +- .../email/mailbox-live-mirror.node.test.ts | 445 +++++++++++++++++ .../worker/src/email/mailbox-live-mirror.ts | 207 ++++++++ .../src/email/mailbox-mirror.node.test.ts | 95 +++- packages/worker/src/email/mailbox-mirror.ts | 464 ++++++++++++++---- .../email/mailbox-parity-events.node.test.ts | 44 +- .../worker/src/email/mailbox-parity-events.ts | 8 +- .../email/mailbox-snapshot-repo.node.test.ts | 55 +++ .../worker/src/email/mailbox-snapshot-repo.ts | 79 ++- packages/worker/src/email/outbound.ts | 50 +- .../worker/src/email/outbound.workers.test.ts | 242 +++++++++ .../email/repo-body-limits.workers.test.ts | 65 ++- packages/worker/src/email/repo.ts | 61 ++- .../email/email-message-classify.node.test.ts | 43 ++ .../email/email-message-classify.ts | 7 + 21 files changed, 2216 insertions(+), 213 deletions(-) create mode 100644 packages/worker/src/email/mailbox-live-mirror.node.test.ts create mode 100644 packages/worker/src/email/mailbox-live-mirror.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 1f2eb593bd..492f2bdb56 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -218,10 +218,11 @@ Durable Object export behavior: - `Mailbox` exports per-user email metadata (threads, messages, attachments, delivery events) through the account-export `mailbox` section (`exportMailbox` RPC; keyset pagination with prefixed cursors over those tables). Manifest - counts use `countMailbox`. Phase 1 registers this consumption; phase 2a adds - dual-write primitives without live callers or a change of D1 authority — D1 - `email_*` rows remain the live source of truth and are still exported in the - `d1` section. See [Mailbox](#durable-objects-mailbox). + counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires + non-inbound live dual-write for outbound terminals, provider-recorded delivery + events, and user classification without changing D1 authority — D1 `email_*` + rows remain the live source of truth and are still exported in the `d1` + section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -376,11 +377,12 @@ When Analytics Engine SQL is unreachable, these two charts zero-fill while the rest of the page renders. Local development uses the existing D1 counters and delivery-event table because Wrangler's emulated dataset has no SQL API. -**Mailbox expand-phase parity events** (additive; no live writers in the -phase-2a PR) reuse the same `EMAIL_EVENTS` dataset with a separate row shape -defined in `packages/worker/src/email/mailbox-parity-events.ts` -(`recordMailboxParityEvent`). These rows are for future dual-write observability -and reconciliation — they do not feed the admin insights charts today: +**Mailbox expand-phase parity events** reuse the same `EMAIL_EVENTS` dataset +with a separate row shape defined in +`packages/worker/src/email/mailbox-parity-events.ts` +(`recordMailboxParityEvent`). Live mirror helpers record one namespaced outcome +per attempted operation; scheduled parity compares are not wired yet. These rows +do not feed the admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) - `blob1` — namespaced event type (`mailbox_mirror:` or @@ -395,10 +397,12 @@ and reconciliation — they do not feed the admin insights charts today: Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, `update_message_delivery`, `set_message_classification`, `delete_message_metadata`, `delete_delivery_event`, `delete_thread_if_empty`. -Mirror outcomes: `mirrored`, `stale`, `skipped`, `error`. Parity operations: -`compare_threads`, `compare_messages`, `compare_attachments`, -`compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Writes are -best-effort and never throw into D1 authority paths. +Mirror outcomes: `mirrored`, `stale`, `missing`, `timeout`, `skipped`, `error`. +Parity operations: `compare_threads`, `compare_messages`, `compare_attachments`, +`compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Mirror outcome +writes are automatic from live dual-write helpers; parity compare writers remain +for future reconciliation. Writes are best-effort and never throw into D1 +authority paths. Two D1 reporting projections deliberately remain: @@ -681,9 +685,10 @@ clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment `storageKey` values against the canonical builders for that `ownerId`. -**Partial mutation RPCs** (phase 2a; owner-bound, monotonic on `updatedAt`, no -R2 or retention-alarm side effects — unlike full snapshot mirrors, these do not -mark retention dirty or reschedule alarms): +**Partial mutation RPCs** (library-only in the non-inbound phase-2 slice; +owner-bound, monotonic on `updatedAt`, no R2 or retention-alarm side effects — +unlike full snapshot mirrors, these do not mark retention dirty or reschedule +alarms): - `touchThread` — advance `last_message_at` / `updated_at` without a full snapshot; `last_message_at` never moves backward @@ -725,20 +730,51 @@ D1 mutation (inserts use `created_at`). Callers must not stitch promoted columns field-by-field. **Best-effort mirror helpers** (`mailbox-mirror.ts`): non-throwing wrappers -around the DO RPCs for future D1-authoritative dual-write. Each returns a -structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, -`{ status: 'stale' }`, `{ status: 'skipped', reason }` (`system-email` | -`mailbox-unconfigured` | `missing-owner`), or `{ status: 'error', error }`. -Failures log with stable tags (for example `mailbox-mirror-message-failed`) and -never propagate into D1 commit paths. Helpers cover full snapshots -(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot` — prefer -loading delivery events with `getMailboxDeliveryEventMirrorInput`) and partial -mutations (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, +around the DO RPCs for D1-authoritative dual-write. Each awaited DO RPC is +bounded by `mailboxMirrorRpcTimeoutMs` (1 second). Each returns a structured +`MailboxMirrorResult`: `{ status: 'mirrored' }`, `{ status: 'stale' }`, +`{ status: 'missing' }`, `{ status: 'timeout' }`, +`{ status: 'skipped', reason }` (`system-email` | `mailbox-unconfigured` | +`missing-owner`), or `{ status: 'error', error }`. Every outcome is recorded +automatically via `recordMailboxParityEvent` (`mailbox_mirror:`) when +a user id is known; `system:email` is excluded. Failures log with stable tags +(for example `mailbox-mirror-message-failed`) and never propagate into D1 commit +paths. Helpers cover full snapshots (`mirrorMailboxMessageSnapshot`, +`mirrorMailboxDeliveryEventSnapshot` — prefer loading delivery events with +`getMailboxDeliveryEventMirrorInput`) and partial mutations +(`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, -`mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). **There -are no live dual-write callers in the phase-2a PR** — only unit/worker tests -invoke these helpers. D1 remains sole authority for all live mail read/write -paths. +`mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). Partial +mutation helpers remain library-only; live paths prefer the graph orchestrator +below. D1 remains sole authority for all live mail read/write paths. + +**Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 +message graph (optional caller thread, message, attachments, then delivery +events) and mirrors it best-effort. `mirrorMailboxMessageGraphFromD1` settles +the message snapshot RPC first, then fans out delivery-event snapshot RPCs +concurrently so event-phase wall time stays near one RPC timeout (not limit × +timeout). Event load uses `max+1` rows so `eventsTruncated` is explicit; at most +`mailboxLiveMirrorMaxEvents` (`mailboxDefaultPageSize`, 100) events are mirrored +per call to stay under Analytics Engine's ~250 writes/request budget (1 message +outcome + up to 100 event outcomes). Never throws; returns a bounded summary. +**Live callers (non-inbound phase-2 slice):** + +- **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound + state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full + message/thread/attachment/event graph. Passes a just-created thread when + present; otherwise the orchestrator loads it from D1. Failures never affect + send/refund. +- **Provider-recorded delivery events** (`delivery-queue.ts`) — when queue + ingestion persists a new provider event (`recorded`), schedules + `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack is not + blocked; full graph repair, not a partial delivery-event touch. +- **User classification** (`account-email.ts`, `email-message-classify.ts`) — + after D1 classification update, mirrors the full graph (not the partial + `setMessageClassification` RPC alone). + +Inbound commit/touch/classify paths, retention deletes, and partial mutation +mirrors are **not** wired in this slice. Parity compare/reconcile cron remains +future work. **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -760,17 +796,21 @@ mail content. Account export pages Mailbox state through the `mailbox` section ### Expand/contract phases -This is an expand/contract migration. **The current PR is phase 2a — -prerequisite scaffolding within phase 2:** additive dual-write **primitives** -(email-owner constant, D1→Mailbox snapshot adapters, best-effort mirror helpers -with structured outcomes, owner-bound partial-mutation and stale-safe -metadata-delete RPCs, and the Analytics Engine parity event schema). It does -**not** wire live dual-write callers, does **not** change live email read/write -authority, and leaves D1 `email_*` tables authoritative for all live paths. D1 -email rows and existing R2 inventory deletion stay authoritative during expand. -**Phase 2 is complete only when every user-mail mutation that writes D1 also -writes the DO and parity/reconciliation are wired** — this PR ships the library -surface that later live-path wiring will call. +This is an expand/contract migration. **Phase 2 is partially wired (non-inbound +slice):** live dual-write covers outbound terminal +message/thread/attachment/event graphs, provider-recorded delivery events (queue +`waitUntil` full graph repair), and user classification (full graph repair after +D1 update). Mirror helpers emit automatic namespaced outcome telemetry +(`missing` and `timeout` included). Every awaited mirror RPC is bounded to 1s; +graph event fan-out is concurrent with explicit truncation when D1 has more than +`mailboxLiveMirrorMaxEvents` delivery events. **`system:email` stays in D1 +only.** Inbound lifecycle dual-write, retention/metadata deletes, and scheduled +parity reconcile are **not** wired in this slice. Live email read/write +authority is unchanged — D1 `email_*` tables remain authoritative for all live +paths. D1 email rows and existing R2 inventory deletion stay authoritative +during expand. **Phase 2 completes only when every user-mail mutation that +writes D1 also writes the DO and parity/reconciliation are wired** — inbound +paths and parity reconcile remain. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -779,15 +819,18 @@ surface that later live-path wiring will call. authority for live mail. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO - completeness against D1. D1 stays authoritative for reads. Phase 2a (current - PR) adds `email-owner.ts`, `mailbox-types.ts`, `mailbox-snapshots.ts`, - `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-mutations.ts` + - matching DO RPCs (`touchThread`, `updateMessageDelivery`, - `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, - `deleteThreadIfEmpty`), and `mailbox-parity-events.ts`. Those helpers and - parity-event writers are library-only today — **no production path invokes - them yet**; live-path dual-write and parity wiring remain for the rest of - phase 2. + completeness against D1. D1 stays authoritative for reads. Library surface: + `email-owner.ts`, `mailbox-types.ts`, `mailbox-snapshots.ts`, + `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-live-mirror.ts`, + `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, + `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, + `deleteDeliveryEvent`, `deleteThreadIfEmpty`), and + `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, + provider `recorded` queue events (`waitUntil`), and user classification call + `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` + outcomes automatically. **Still pending for phase-2 completion:** inbound + commit/touch/classify dual-write, retention and metadata-delete mirrors, and + scheduled parity compare/reconcile. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata @@ -835,7 +878,8 @@ metadata store: message (logged, non-retry) rather than risking a duplicate on Email Routing retry. -Phase 1 and phase 2a scaffolding does not change that live D1 boundary. +Phase 1 scaffolding and the non-inbound phase-2 live slice do not change that +live D1 inbound boundary; inbound dual-write remains pending phase-2 completion. ### Package state model @@ -956,7 +1000,7 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` reconciliation alarms) - `MAILBOX` (Durable Objects; per-user email metadata — see [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export - consumption; phase 2a adds dual-write primitives without changing D1 + consumption; phase 2 wires non-inbound live dual-write without changing D1 authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) diff --git a/packages/worker/src/app/handlers/account-email.node.test.ts b/packages/worker/src/app/handlers/account-email.node.test.ts index fa111e18b0..e206514d67 100644 --- a/packages/worker/src/app/handlers/account-email.node.test.ts +++ b/packages/worker/src/app/handlers/account-email.node.test.ts @@ -155,6 +155,11 @@ const mockModule = vi.hoisted(() => ({ }, ]), setEmailMessageClassification: vi.fn(async () => true), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'msg-1', + message: { status: 'mirrored' }, + events: [], + })), prepare: vi.fn(), })) @@ -226,6 +231,11 @@ vi.mock('#worker/email/repo.ts', () => ({ mockModule.setEmailMessageClassification(...args), })) +vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: (...args: Array) => + mockModule.mirrorMailboxMessageGraphFromD1(...args), +})) + const { createAccountEmailApiHandler } = await import('./account-email.ts') function createCountResult(total: number) { @@ -558,6 +568,7 @@ test('email API lists classification filters and classifies inbound messages', a }) mockModule.setEmailMessageClassification.mockClear() + mockModule.mirrorMailboxMessageGraphFromD1.mockClear() // Reload after classify uses the default message list again. createEnv({ meter, messageRows: [messageRow] }) const quarantineResponse = await handler.handler({ @@ -579,12 +590,19 @@ test('email API lists classification filters and classifies inbound messages', a classification: 'quarantined', classificationReason: 'Reclassified by user.', }) + expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'stable-user-1', + messageId: 'msg-1', + }) await expect(quarantineResponse.json()).resolves.toMatchObject({ ok: true, selectedMessage: expect.objectContaining({ id: 'msg-1' }), }) mockModule.setEmailMessageClassification.mockClear() + mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const acceptResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -604,8 +622,15 @@ test('email API lists classification filters and classifies inbound messages', a classification: 'accepted', classificationReason: null, }) + expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'stable-user-1', + messageId: 'msg-1', + }) mockModule.setEmailMessageClassification.mockResolvedValueOnce(false) + mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const missingResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -618,6 +643,33 @@ test('email API lists classification filters and classifies inbound messages', a }), }) expect(missingResponse.status).toBe(404) + expect(mockModule.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + + mockModule.setEmailMessageClassification.mockResolvedValueOnce(true) + mockModule.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'msg-1', + message: { + status: 'error', + error: new Error('mailbox mirror timed out'), + }, + events: [], + }) + const mirrorFailureResponse = await handler.handler({ + request: new Request('https://example.com/account/email.json', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + action: 'classify', + message_id: 'msg-1', + classification: 'quarantined', + }), + }), + }) + expect(mirrorFailureResponse.status).toBe(200) + await expect(mirrorFailureResponse.json()).resolves.toMatchObject({ + ok: true, + selectedMessage: expect.objectContaining({ id: 'msg-1' }), + }) const invalidActionResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { diff --git a/packages/worker/src/app/handlers/account-email.ts b/packages/worker/src/app/handlers/account-email.ts index aa34b19234..62b8148201 100644 --- a/packages/worker/src/app/handlers/account-email.ts +++ b/packages/worker/src/app/handlers/account-email.ts @@ -7,6 +7,7 @@ import { requireAuthenticatedPageUser } from '#app/page-auth.ts' import { readTrimmedStringOrEmpty } from '#app/request-body.ts' import { type routes } from '#app/routes.ts' import { renderAppPage } from '#app/ssr-render.tsx' +import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' import { setEmailMessageClassification } from '#worker/email/repo.ts' import { emailClassificationValues, @@ -129,6 +130,13 @@ export function createAccountEmailApiHandler(env: Env) { ) } + await mirrorMailboxMessageGraphFromD1({ + env, + db: env.APP_DB, + userId: user.mcpUser.userId, + messageId, + }) + const listUrl = new URL(request.url, 'http://localhost') listUrl.searchParams.set('selected', messageId) return jsonResponse( diff --git a/packages/worker/src/email/delivery-events.ts b/packages/worker/src/email/delivery-events.ts index 4f92ee6742..19aa3f8a10 100644 --- a/packages/worker/src/email/delivery-events.ts +++ b/packages/worker/src/email/delivery-events.ts @@ -94,30 +94,35 @@ export async function processCloudflareEmailDeliveryEvent(input: { reportingEnv?: EmailReportingEnv body: unknown }) { - const event = parseCloudflareEmailDeliveryEvent(input.body) - if (!event) { - return { outcome: 'invalid' as const, event: null, message: null } + const providerEvent = parseCloudflareEmailDeliveryEvent(input.body) + if (!providerEvent) { + return { + outcome: 'invalid' as const, + providerEvent: null, + event: null, + message: null, + } } const result = await recordProviderEmailDeliveryEvent({ db: input.db, reportingEnv: input.reportingEnv, - providerMessageId: event.payload.messageId, - providerEventId: event.payload.eventId, - deliveryStatus: event.payload.delivery.status, - eventTimestamp: event.metadata.eventTimestamp, + providerMessageId: providerEvent.payload.messageId, + providerEventId: providerEvent.payload.eventId, + deliveryStatus: providerEvent.payload.delivery.status, + eventTimestamp: providerEvent.metadata.eventTimestamp, detail: { - source: event.source, - sender: event.payload.sender, - recipient: event.payload.recipient, - subject: event.payload.subject ?? null, - terminal: event.payload.terminal, - delivery: event.payload.delivery, - bounce: event.payload.bounce ?? null, - failure: event.payload.failure ?? null, - rejection: event.payload.rejection ?? null, - complaint: event.payload.complaint ?? null, - metadata: event.metadata, + source: providerEvent.source, + sender: providerEvent.payload.sender, + recipient: providerEvent.payload.recipient, + subject: providerEvent.payload.subject ?? null, + terminal: providerEvent.payload.terminal, + delivery: providerEvent.payload.delivery, + bounce: providerEvent.payload.bounce ?? null, + failure: providerEvent.payload.failure ?? null, + rejection: providerEvent.payload.rejection ?? null, + complaint: providerEvent.payload.complaint ?? null, + metadata: providerEvent.metadata, }, }) - return { ...result, event } + return { ...result, providerEvent } } diff --git a/packages/worker/src/email/delivery-events.workers.test.ts b/packages/worker/src/email/delivery-events.workers.test.ts index 2e0957705f..e218031520 100644 --- a/packages/worker/src/email/delivery-events.workers.test.ts +++ b/packages/worker/src/email/delivery-events.workers.test.ts @@ -1,11 +1,13 @@ import { env } from 'cloudflare:workers' import { expect, test } from 'vitest' import { processCloudflareEmailDeliveryEvent } from './delivery-events.ts' +import { mirrorMailboxMessageGraphFromD1 } from './mailbox-live-mirror.ts' import { getEmailMessageById, insertEmailMessage, listEmailDeliveryEvents, } from './repo.ts' +import { mailboxEnv, rpcFor } from './mailbox-test-helpers.ts' import { ensureEmailTestSchema } from './test-schema.ts' function createDeliveryEvent(input: { @@ -90,6 +92,12 @@ test('provider delivery events are idempotent, ordered, and user scoped', async body: delivered, }) expect(recorded.outcome).toBe('recorded') + expect(recorded.event).toMatchObject({ + messageId: message.id, + userId, + eventType: 'delivered', + providerEventId: 'event-delivered', + }) expect(recorded.message).toMatchObject({ id: message.id, userId, @@ -209,3 +217,135 @@ test('provider delivery events are idempotent, ordered, and user scoped', async ).outcome, ).toBe('invalid') }) + +test('recorded provider delivery events mirror the D1 message graph to Mailbox when configured', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = `delivery-mailbox-${crypto.randomUUID()}` + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'Mailbox mirror', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-07-17T20:00:00.000Z', + }, + }) + const mailbox = rpcFor(userId) + // Warm the per-user DO before graph mirror (RPC timeout is ~1s). + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const recorded = await processCloudflareEmailDeliveryEvent({ + db: env.APP_DB, + body: createDeliveryEvent({ + eventId: 'event-mailbox-delivered', + messageId: providerMessageId, + status: 'delivered', + eventTimestamp: '2026-07-17T20:02:00.000Z', + }), + }) + expect(recorded.outcome).toBe('recorded') + if (!recorded.message) { + throw new Error('Expected recorded provider delivery message.') + } + + const summary = await mirrorMailboxMessageGraphFromD1({ + env: { ...env, ...mailboxEnv() }, + db: env.APP_DB, + userId, + messageId: message.id, + }) + expect(summary).toMatchObject({ + messageId: message.id, + message: { status: 'mirrored' }, + eventsTruncated: false, + }) + expect(summary.events).toEqual([ + expect.objectContaining({ + eventId: recorded.event!.id, + result: { status: 'mirrored' }, + }), + ]) + + const events = await mailbox.listDeliveryEvents({ + messageId: message.id, + limit: 10, + }) + expect(events).toHaveLength(1) + expect(events[0]).toMatchObject({ + eventType: 'delivered', + provider: 'cloudflare-email', + providerEventId: 'event-mailbox-delivered', + providerMessageId, + }) + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-17T20:02:00.000Z', + }) + expect( + await getEmailMessageById({ + db: env.APP_DB, + userId, + messageId: message.id, + }), + ).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-17T20:02:00.000Z', + }) +}, 30_000) + +test('recorded provider delivery graph mirror skips cleanly when MAILBOX is absent', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = `delivery-no-mailbox-${crypto.randomUUID()}` + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'No mailbox binding', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-07-17T20:00:00.000Z', + }, + }) + + const recorded = await processCloudflareEmailDeliveryEvent({ + db: env.APP_DB, + body: createDeliveryEvent({ + eventId: 'event-no-mailbox', + messageId: providerMessageId, + status: 'delivered', + eventTimestamp: '2026-07-17T20:02:00.000Z', + }), + }) + expect(recorded.outcome).toBe('recorded') + if (!recorded.message) { + throw new Error('Expected recorded provider delivery message.') + } + + expect( + await mirrorMailboxMessageGraphFromD1({ + env: { APP_DB: env.APP_DB }, + db: env.APP_DB, + userId, + messageId: message.id, + }), + ).toEqual({ + messageId: message.id, + message: { status: 'skipped', reason: 'mailbox-unconfigured' }, + events: [ + { + eventId: recorded.event!.id, + result: { status: 'skipped', reason: 'mailbox-unconfigured' }, + }, + ], + eventsTruncated: false, + }) +}) diff --git a/packages/worker/src/email/delivery-queue.node.test.ts b/packages/worker/src/email/delivery-queue.node.test.ts index 6c4e64d044..8030c12213 100644 --- a/packages/worker/src/email/delivery-queue.node.test.ts +++ b/packages/worker/src/email/delivery-queue.node.test.ts @@ -7,6 +7,11 @@ import { const mocks = vi.hoisted(() => ({ processCloudflareEmailDeliveryEvent: vi.fn(), dispatchEmailDeliverySubscriptionEvents: vi.fn(async () => []), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'message-1', + message: { status: 'mirrored' as const }, + events: [], + })), })) vi.mock('./delivery-events.ts', () => ({ @@ -14,6 +19,10 @@ vi.mock('./delivery-events.ts', () => ({ mocks.processCloudflareEmailDeliveryEvent, })) +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, +})) + vi.mock('./package-subscriptions.ts', () => ({ dispatchEmailDeliverySubscriptionEvents: mocks.dispatchEmailDeliverySubscriptionEvents, @@ -53,35 +62,50 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched }, } const storedMessage = { id: 'message-1', userId: 'user-1' } + const d1Event = { + id: 'd1-event-1', + messageId: 'message-1', + userId: 'user-1', + inboxId: 'inbox-1', + eventType: 'delivered', + provider: 'cloudflare-email', + providerMessageId: 'provider-1', + providerEventId: 'event-1', + detailJson: '{}', + createdAt: '2026-07-17T20:02:00.000Z', + } mocks.processCloudflareEmailDeliveryEvent .mockResolvedValueOnce({ outcome: 'recorded', - event: providerEvent, + providerEvent, + event: d1Event, message: storedMessage, }) .mockResolvedValueOnce({ outcome: 'duplicate', - event: providerEvent, + providerEvent, message: storedMessage, }) .mockResolvedValueOnce({ outcome: 'invalid', + providerEvent: null, event: null, message: null, }) .mockResolvedValueOnce({ outcome: 'stale', - event: providerEvent, + providerEvent, message: storedMessage, }) .mockResolvedValueOnce({ outcome: 'unmatched', - event: providerEvent, + providerEvent, message: null, }) .mockResolvedValueOnce({ outcome: 'recorded', - event: providerEvent, + providerEvent, + event: d1Event, message: storedMessage, }) mocks.dispatchEmailDeliverySubscriptionEvents @@ -135,6 +159,14 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched waitUntil: expect.any(Function), }, ) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: expect.anything(), + db: expect.anything(), + userId: 'user-1', + messageId: 'message-1', + }) + expect(waitUntilPromises).toHaveLength(1) expect(consoleWarn).toHaveBeenCalledWith('email-delivery-event-unmatched', { queueMessageId: 'queue-unmatched', providerMessageId: 'provider-1', @@ -144,3 +176,117 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched expect.objectContaining({ message: expect.stringContaining('KV failure') }), ) }) + +test('recorded delivery events schedule mailbox graph mirror via waitUntil without blocking ack', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockResolvedValue([]) + + const providerEvent = { + payload: { + eventId: 'event-mirror', + messageId: 'provider-mirror', + delivery: { status: 'delivered' }, + }, + } + const storedMessage = { id: 'message-mirror', userId: 'user-mirror' } + mocks.processCloudflareEmailDeliveryEvent.mockResolvedValue({ + outcome: 'recorded', + providerEvent, + event: { id: 'd1-event-mirror' }, + message: storedMessage, + }) + + let resolveMirror: (() => void) | undefined + const mirrorPromise = new Promise<{ messageId: string }>((resolve) => { + resolveMirror = () => + resolve({ + messageId: 'message-mirror', + message: { status: 'mirrored' }, + events: [], + }) + }) + mocks.mirrorMailboxMessageGraphFromD1.mockReturnValue(mirrorPromise) + + const queueMessage = createQueueMessage('queue-mirror', { kind: 'recorded' }) + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue( + { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch, + { APP_DB: {} } as Env, + ctx, + ) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).not.toHaveBeenCalled() + expect(waitUntilPromises).toHaveLength(1) + resolveMirror?.() + await Promise.all(waitUntilPromises) +}) + +test('mailbox graph mirror failures scheduled in waitUntil do not reject or alter queue ack', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockResolvedValue([]) + + const providerEvent = { + payload: { + eventId: 'event-fail', + messageId: 'provider-fail', + delivery: { status: 'delivered' }, + }, + } + const storedMessage = { id: 'message-fail', userId: 'user-fail' } + mocks.processCloudflareEmailDeliveryEvent.mockResolvedValue({ + outcome: 'recorded', + providerEvent, + event: { id: 'd1-event-fail' }, + message: storedMessage, + }) + mocks.mirrorMailboxMessageGraphFromD1.mockRejectedValue( + new Error('mailbox rpc unavailable'), + ) + + const queueMessage = createQueueMessage('queue-mirror-fail', { + kind: 'recorded', + }) + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue( + { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch, + { APP_DB: {} } as Env, + ctx, + ) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).not.toHaveBeenCalled() + await expect(Promise.all(waitUntilPromises)).rejects.toThrow( + 'mailbox rpc unavailable', + ) +}) diff --git a/packages/worker/src/email/delivery-queue.ts b/packages/worker/src/email/delivery-queue.ts index f79ceea6b6..bb843ae987 100644 --- a/packages/worker/src/email/delivery-queue.ts +++ b/packages/worker/src/email/delivery-queue.ts @@ -1,4 +1,5 @@ import { processCloudflareEmailDeliveryEvent } from './delivery-events.ts' +import { mirrorMailboxMessageGraphFromD1 } from './mailbox-live-mirror.ts' import { applyOutboundEmailAbusePause } from './outbound-abuse.ts' import { dispatchEmailDeliverySubscriptionEvents } from './package-subscriptions.ts' import { type EmailReportingEnv } from './reporting-events.ts' @@ -26,7 +27,7 @@ export async function handleEmailDeliveryQueue( case 'unmatched': console.warn('email-delivery-event-unmatched', { queueMessageId: queueMessage.id, - providerMessageId: result.event?.payload.messageId ?? null, + providerMessageId: result.providerEvent?.payload.messageId ?? null, }) queueMessage.retry({ delaySeconds: unmatchedRetryDelaySeconds }) break @@ -39,7 +40,7 @@ export async function handleEmailDeliveryQueue( await applyOutboundEmailAbusePause({ env, userId: result.message.userId, - deliveryStatus: result.event.payload.delivery.status, + deliveryStatus: result.providerEvent.payload.delivery.status, eventRecorded: false, }) queueMessage.ack() @@ -53,15 +54,25 @@ export async function handleEmailDeliveryQueue( await applyOutboundEmailAbusePause({ env, userId: result.message.userId, - deliveryStatus: result.event.payload.delivery.status, + deliveryStatus: result.providerEvent.payload.delivery.status, eventRecorded: result.outcome === 'recorded', }) await dispatchEmailDeliverySubscriptionEvents({ env, message: result.message, - providerEvent: result.event, + providerEvent: result.providerEvent, waitUntil, }) + if (result.outcome === 'recorded' && result.message != null) { + waitUntil( + mirrorMailboxMessageGraphFromD1({ + env, + db: env.APP_DB, + userId: result.message.userId, + messageId: result.message.id, + }), + ) + } queueMessage.ack() break } diff --git a/packages/worker/src/email/mailbox-live-mirror.node.test.ts b/packages/worker/src/email/mailbox-live-mirror.node.test.ts new file mode 100644 index 0000000000..af8ae5ba64 --- /dev/null +++ b/packages/worker/src/email/mailbox-live-mirror.node.test.ts @@ -0,0 +1,445 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import { + mailboxLiveMirrorMaxAnalyticsWrites, + mailboxLiveMirrorMaxEvents, + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, +} from './mailbox-live-mirror.ts' +import { ensureEmailTestSchema } from './test-schema.ts' +import { type EmailThreadRecord } from './types.ts' + +async function createEmailDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +function fakeMailboxEnv(stub: Record) { + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const get = vi.fn(() => stub) + const writeDataPoint = vi.fn() + return { + env: { + MAILBOX: { + idFromName, + get, + } as unknown as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, + }, + writeDataPoint, + } +} + +async function seedMessage(db: D1Database) { + const at = '2026-07-01T12:00:00.000Z' + await db + .prepare( + `INSERT INTO email_threads ( + id, user_id, inbox_id, subject_normalized, root_message_id_header, + last_message_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'thread-1', + 'user-aaa', + 'inbox-1', + 'hello', + '', + at, + at, + at, + ) + .run() + await db + .prepare( + `INSERT INTO email_messages ( + id, direction, user_id, inbox_id, thread_id, from_address, + to_addresses_json, subject, raw_mime_key, raw_size, + processing_status, classification, received_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'msg-1', + 'outbound', + 'user-aaa', + 'inbox-1', + 'thread-1', + 'sender@example.com', + '["owner@example.com"]', + 'Hello', + null, + 0, + 'sent', + 'accepted', + null, + at, + at, + ) + .run() + await db + .prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .bind('att-1', 'msg-1', 'note.txt', 'text/plain', 4, 'unavailable', at) + .run() +} + +async function seedDeliveryEvent( + db: D1Database, + input: { id: string; eventType: string; createdAt: string }, +) { + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + input.id, + 'msg-1', + 'user-aaa', + 'inbox-1', + input.eventType, + 'kody', + null, + null, + '{}', + 0, + input.createdAt, + ) + .run() +} + +async function seedMessageGraph(db: D1Database) { + await seedMessage(db) + await seedDeliveryEvent(db, { + id: 'evt-1', + eventType: 'send_requested', + createdAt: '2026-07-01T12:00:01.000Z', + }) + await seedDeliveryEvent(db, { + id: 'evt-2', + eventType: 'sent', + createdAt: '2026-07-01T12:00:02.000Z', + }) +} + +test('live-mirror AE write budget stays under Analytics Engine request cap', () => { + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe( + 1 + mailboxLiveMirrorMaxEvents, + ) +}) + +test('live-mirror graph mirrors message before events and uses createdAt for event updatedAt', async () => { + const { db } = await createEmailDb() + await seedMessageGraph(db) + + let messageSettled = false + const mirrorMessage = vi.fn(async () => { + messageSettled = true + return { ok: true as const, accepted: true } + }) + const upsertDeliveryEvent = vi.fn( + async (input: { event: { id: string } }) => { + expect(messageSettled).toBe(true) + return { + inserted: true, + accepted: true, + updatedLatestStatus: false, + eventId: input.event.id, + } + }, + ) + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) + + const thread: EmailThreadRecord = { + id: 'thread-1', + userId: 'user-aaa', + inboxId: 'inbox-1', + subjectNormalized: 'hello', + rootMessageIdHeader: '', + lastMessageAt: '2026-07-01T12:00:00.000Z', + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + } + + const summary = await mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + sourceMutationAt: '2026-07-01T12:00:03.000Z', + thread, + }) + + expect(summary).toEqual({ + messageId: 'msg-1', + message: { status: 'mirrored' }, + events: [ + { eventId: 'evt-1', result: { status: 'mirrored' } }, + { eventId: 'evt-2', result: { status: 'mirrored' } }, + ], + eventsTruncated: false, + }) + expect(mirrorMessage).toHaveBeenCalledWith( + expect.objectContaining({ + ownerId: 'user-aaa', + thread: expect.objectContaining({ id: 'thread-1' }), + message: expect.objectContaining({ id: 'msg-1' }), + attachments: [ + expect.objectContaining({ id: 'att-1', filename: 'note.txt' }), + ], + }), + ) + + // When thread is omitted, load it from D1 via getEmailThreadById. + mirrorMessage.mockClear() + upsertDeliveryEvent.mockClear() + await mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + expect(mirrorMessage).toHaveBeenCalledWith( + expect.objectContaining({ + thread: expect.objectContaining({ + id: 'thread-1', + subjectNormalized: 'hello', + }), + }), + ) + expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + id: 'evt-1', + createdAt: '2026-07-01T12:00:01.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }), + }), + ) + expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + id: 'evt-2', + createdAt: '2026-07-01T12:00:02.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + }), + ) +}) + +test('live-mirror graph fans out event RPCs concurrently after message settles', async () => { + vi.useFakeTimers() + try { + const { db } = await createEmailDb() + await seedMessage(db) + const eventCount = 5 + for (let index = 0; index < eventCount; index += 1) { + await seedDeliveryEvent(db, { + id: `evt-${index + 1}`, + eventType: index === 0 ? 'send_requested' : 'sent', + createdAt: `2026-07-01T12:00:0${index + 1}.000Z`, + }) + } + + const eventDelayMs = 400 + let maxInFlight = 0 + let inFlight = 0 + let messageSettled = false + const mirrorMessage = vi.fn(async () => { + messageSettled = true + return { ok: true as const, accepted: true } + }) + const upsertDeliveryEvent = vi.fn(async () => { + expect(messageSettled).toBe(true) + inFlight += 1 + maxInFlight = Math.max(maxInFlight, inFlight) + await new Promise((resolve) => { + setTimeout(resolve, eventDelayMs) + }) + inFlight -= 1 + return { inserted: true, accepted: true, updatedLatestStatus: false } + }) + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) + + const pending = mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + + // One event-delay tick completes the whole fan-out (not eventCount × delay). + await vi.advanceTimersByTimeAsync(eventDelayMs) + const summary = await pending + + expect(summary.events).toHaveLength(eventCount) + expect(summary.eventsTruncated).toBe(false) + expect(maxInFlight).toBe(eventCount) + expect(upsertDeliveryEvent).toHaveBeenCalledTimes(eventCount) + // Wall budget stays near one RPC timeout for the event phase. + expect(eventDelayMs).toBeLessThan(mailboxMirrorRpcTimeoutMs) + } finally { + vi.useRealTimers() + } +}) + +test('live-mirror graph sets eventsTruncated and mirrors at most max events', async () => { + const { db } = await createEmailDb() + await seedMessage(db) + + const totalEvents = mailboxLiveMirrorMaxEvents + 1 + const baseMs = Date.parse('2026-07-01T12:00:00.000Z') + for (let index = 0; index < totalEvents; index += 1) { + const id = `evt-${String(index).padStart(3, '0')}` + await seedDeliveryEvent(db, { + id, + eventType: index === 0 ? 'send_requested' : 'sent', + createdAt: new Date(baseMs + index * 1000).toISOString(), + }) + } + + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const { env, writeDataPoint } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvent, + }) + + const summary = await mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + + expect(summary.eventsTruncated).toBe(true) + expect(summary.events).toHaveLength(mailboxLiveMirrorMaxEvents) + expect(summary.events[0]?.eventId).toBe('evt-000') + expect(summary.events.at(-1)?.eventId).toBe( + `evt-${String(mailboxLiveMirrorMaxEvents - 1).padStart(3, '0')}`, + ) + expect(upsertDeliveryEvent).toHaveBeenCalledTimes(mailboxLiveMirrorMaxEvents) + // 1 message + max events AE writes; never the overflow row. + expect(writeDataPoint).toHaveBeenCalledTimes( + mailboxLiveMirrorMaxAnalyticsWrites, + ) + expect(writeDataPoint.mock.calls.length).toBeLessThan(250) +}) + +test('live-mirror graph returns missing when D1 message is absent', async () => { + const { db } = await createEmailDb() + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const { env } = fakeMailboxEnv({ mirrorMessage }) + + await expect( + mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'missing-msg', + }), + ).resolves.toEqual({ + messageId: 'missing-msg', + message: { status: 'missing' }, + events: [], + eventsTruncated: false, + }) + expect(mirrorMessage).not.toHaveBeenCalled() +}) + +test('live-mirror graph catches unexpected errors without throwing', async () => { + consoleWarn.mockImplementation(() => {}) + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const { env } = fakeMailboxEnv({ mirrorMessage }) + const db = { + prepare() { + throw new Error('d1 exploded') + }, + } as unknown as D1Database + + await expect( + mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }), + ).resolves.toEqual({ + messageId: 'msg-1', + message: { + status: 'error', + error: expect.objectContaining({ message: 'd1 exploded' }), + }, + events: [], + eventsTruncated: false, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-live-mirror-message-graph-failed', + expect.objectContaining({ message: 'd1 exploded' }), + ) +}) + +test('live-mirror one-event helper loads cohesive projection and returns missing', async () => { + const { db } = await createEmailDb() + await seedMessageGraph(db) + + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const { env } = fakeMailboxEnv({ upsertDeliveryEvent }) + + expect( + await mirrorMailboxDeliveryEventFromD1({ + env, + db, + userId: 'user-aaa', + eventId: 'evt-2', + }), + ).toEqual({ status: 'mirrored' }) + expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + id: 'evt-2', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + }), + ) + + expect( + await mirrorMailboxDeliveryEventFromD1({ + env, + db, + userId: 'user-aaa', + eventId: 'missing-evt', + }), + ).toEqual({ status: 'missing' }) +}) diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts new file mode 100644 index 0000000000..062f915b8b --- /dev/null +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -0,0 +1,207 @@ +import { + mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxMessageSnapshot, + type MailboxMirrorEnv, + type MailboxMirrorResult, +} from './mailbox-mirror.ts' +import { + getEmailDeliveryEventMirrorProjection, + listMailboxDeliveryEventMirrorInputsForMessage, +} from './mailbox-snapshot-repo.ts' +import { toMailboxDeliveryEventInput } from './mailbox-snapshots.ts' +import { mailboxDefaultPageSize } from './mailbox-types.ts' +import { + getEmailMessageById, + getEmailThreadById, + listEmailAttachmentsForMessage, +} from './repo.ts' +import { type EmailThreadRecord } from './types.ts' + +/** + * High-level D1 → Mailbox live-mirror orchestration. + * + * D1 remains authoritative. These helpers load cohesive snapshots from D1 and + * call best-effort mirror RPCs; they never throw into caller paths and do not + * perform deletes or inbound-lifecycle special cases. + * + * Callers that just created a thread may pass it via `thread` to avoid a + * round-trip; otherwise the graph helper loads it with `getEmailThreadById`. + * + * Graph event fan-out runs concurrently after the message RPC settles so event + * wall time stays near one RPC timeout (not limit × timeout). Analytics Engine + * parity writes stay under the ~250/request cap: 1 message + max events. + */ + +/** + * Max delivery events mirrored per graph call. + * `1 + mailboxLiveMirrorMaxEvents` AE writes must stay under ~250/request. + */ +export const mailboxLiveMirrorMaxEvents = mailboxDefaultPageSize + +/** AE write budget headroom check (message + event outcomes). */ +export const mailboxLiveMirrorMaxAnalyticsWrites = + 1 + mailboxLiveMirrorMaxEvents + +export type MailboxLiveMirrorEnv = MailboxMirrorEnv + +export type MailboxLiveMirrorEventResult = { + eventId: string + result: MailboxMirrorResult +} + +/** + * Bounded summary for one message graph mirror attempt. + * Always returned; never thrown. + */ +export type MailboxLiveMirrorGraphSummary = { + messageId: string + message: MailboxMirrorResult + events: Array + /** True when D1 had more than {@link mailboxLiveMirrorMaxEvents} events. */ + eventsTruncated: boolean +} + +function emptyGraphSummary( + messageId: string, + message: MailboxMirrorResult, +): MailboxLiveMirrorGraphSummary { + return { + messageId, + message, + events: [], + eventsTruncated: false, + } +} + +/** + * Load one ready delivery-event projection from D1 and mirror it. + * + * When `sourceMutationAt` is omitted, uses the event's `created_at` (immutable + * outbound/provider inserts). Returns `{ status: 'missing' }` when the D1 row + * is absent; never throws. + */ +export async function mirrorMailboxDeliveryEventFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + eventId: string + sourceMutationAt?: string +}): Promise { + try { + const projection = await getEmailDeliveryEventMirrorProjection({ + db: input.db, + ownerId: input.userId, + eventId: input.eventId, + }) + if (!projection) return { status: 'missing' } + const sourceMutationAt = + input.sourceMutationAt != null && input.sourceMutationAt.length > 0 + ? input.sourceMutationAt + : projection.createdAt + return await mirrorMailboxDeliveryEventSnapshot({ + env: input.env, + ownerId: input.userId, + event: toMailboxDeliveryEventInput({ + projection, + sourceMutationAt, + }), + }) + } catch (error) { + console.warn('mailbox-live-mirror-delivery-event-failed', error) + return { status: 'error', error } + } +} + +/** + * Load the authoritative D1 message graph and best-effort mirror it: + * message (+ optional caller-supplied thread, attachments) first, then all + * delivery events concurrently (cohesive projection; `created_at` as + * `sourceMutationAt`). + * + * Never throws. Returns a bounded structured summary. Queries `max+1` events + * so truncation is explicit; mirrors at most {@link mailboxLiveMirrorMaxEvents}. + * + * @param sourceMutationAt - Optional caller mutation timestamp for context; + * D1 row timestamps remain authoritative. Delivery events always use + * `created_at` for immutable outbound/provider inserts. + * @param thread - Optional thread from the caller. When omitted, loads via + * `getEmailThreadById` when `message.threadId` is set. Explicit `null` + * skips the thread snapshot. + */ +export async function mirrorMailboxMessageGraphFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + messageId: string + sourceMutationAt?: string + thread?: EmailThreadRecord | null +}): Promise { + // Caller mutation timestamp is accepted for API symmetry with dual-write + // sites; D1 row timestamps remain authoritative for the loaded graph. + void input.sourceMutationAt + try { + const message = await getEmailMessageById({ + db: input.db, + userId: input.userId, + messageId: input.messageId, + }) + if (!message) { + return emptyGraphSummary(input.messageId, { status: 'missing' }) + } + + const attachments = await listEmailAttachmentsForMessage({ + db: input.db, + messageId: input.messageId, + }) + + let thread = input.thread + if (thread === undefined && message.threadId) { + thread = await getEmailThreadById({ + db: input.db, + userId: input.userId, + threadId: message.threadId, + }) + } + + // Message settles before any event RPC starts (ordering guarantee). + const messageResult = await mirrorMailboxMessageSnapshot({ + env: input.env, + thread, + message, + attachments, + }) + + const loadedEvents = await listMailboxDeliveryEventMirrorInputsForMessage({ + db: input.db, + ownerId: input.userId, + messageId: input.messageId, + limit: mailboxLiveMirrorMaxEvents + 1, + }) + const eventsTruncated = loadedEvents.length > mailboxLiveMirrorMaxEvents + const eventInputs = eventsTruncated + ? loadedEvents.slice(0, mailboxLiveMirrorMaxEvents) + : loadedEvents + + // Concurrent fan-out: event-phase wall time ≈ one RPC timeout, not N×. + const events = await Promise.all( + eventInputs.map(async (event) => ({ + eventId: event.id, + result: await mirrorMailboxDeliveryEventSnapshot({ + env: input.env, + ownerId: input.userId, + event, + }), + })), + ) + + return { + messageId: input.messageId, + message: messageResult, + events, + eventsTruncated, + } + } catch (error) { + console.warn('mailbox-live-mirror-message-graph-failed', error) + return emptyGraphSummary(input.messageId, { status: 'error', error }) + } +} diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 0bc60502e5..0ede393f94 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -2,6 +2,7 @@ import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' import { systemEmailOwnerId } from './email-owner.ts' import { + mailboxMirrorRpcTimeoutMs, mirrorMailboxDeliveryEventSnapshot, mirrorMailboxDeleteDeliveryEvent, mirrorMailboxDeleteMessageMetadata, @@ -17,15 +18,20 @@ import { type MailboxDeliveryEventInput } from './mailbox-types.ts' function fakeMailboxEnv(stub: Record) { const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) const get = vi.fn(() => stub) + const writeDataPoint = vi.fn() return { env: { MAILBOX: { idFromName, get, } as unknown as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint, + } as unknown as AnalyticsEngineDataset, }, idFromName, get, + writeDataPoint, } } @@ -107,7 +113,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep status: 'missing' as const, })) - const { env, idFromName } = fakeMailboxEnv({ + const { env, idFromName, writeDataPoint } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent, touchThread, @@ -266,7 +272,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'stale' }) - // missing partial mutation is idempotent success for best-effort dual-write + // missing partial mutation is a distinct mirror outcome (not mirrored) expect( await mirrorMailboxSetMessageClassification({ env, @@ -276,7 +282,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep classificationReason: 'manual', updatedAt: '2026-07-01T14:00:00.000Z', }), - ).toEqual({ status: 'mirrored' }) + ).toEqual({ status: 'missing' }) expect( await mirrorMailboxDeleteMessageMetadata({ @@ -296,6 +302,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'stale' }) + // delete missing remains mirrored — desired absence is achieved expect( await mirrorMailboxDeleteMessageMetadata({ env, @@ -345,7 +352,7 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep expect( await mirrorMailboxMessageSnapshot({ - env: {}, + env: { EMAIL_EVENTS: env.EMAIL_EVENTS }, message: baseMessage(), }), ).toEqual({ status: 'skipped', reason: 'mailbox-unconfigured' }) @@ -360,4 +367,84 @@ test('mailbox mirror helpers scope by user idFromName, convert payloads, and rep }), ).toEqual({ status: 'skipped', reason: 'missing-owner' }) expect(touchThread).toHaveBeenCalledTimes(1) + + const outcomes = writeDataPoint.mock.calls.map( + (call) => (call[0] as { blobs: Array }).blobs, + ) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:mirror_message', + 'mirrored', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:set_message_classification', + 'missing', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:update_message_delivery', + 'stale', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:delete_delivery_event', + 'error', + expect.any(String), + ]) + expect(outcomes).toContainEqual([ + 'mailbox_mirror:mirror_message', + 'skipped', + expect.any(String), + ]) + // system email must not emit a parity data point + expect( + outcomes.filter( + (blobs) => + blobs[0] === 'mailbox_mirror:mirror_message' && blobs[1] === 'skipped', + ), + ).toHaveLength(1) +}) + +test('mailbox mirror RPC timeout returns timeout and late rejection is not unhandled', async () => { + vi.useFakeTimers() + consoleWarn.mockImplementation(() => {}) + + let rejectRpc!: (error: unknown) => void + const mirrorMessage = vi.fn( + () => + new Promise<{ ok: true; accepted: boolean }>((_resolve, reject) => { + rejectRpc = reject + }), + ) + const { env, writeDataPoint } = fakeMailboxEnv({ mirrorMessage }) + + const pending = mirrorMailboxMessageSnapshot({ + env, + message: baseMessage(), + }) + const expectation = expect(pending).resolves.toEqual({ status: 'timeout' }) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + await expectation + + expect(writeDataPoint).toHaveBeenCalledWith( + expect.objectContaining({ + indexes: ['user-aaa'], + blobs: ['mailbox_mirror:mirror_message', 'timeout', expect.any(String)], + }), + ) + + const unhandled: Array = [] + const onUnhandled = (reason: unknown) => { + unhandled.push(reason) + } + process.on('unhandledRejection', onUnhandled) + try { + rejectRpc(new Error('late rejection after timeout')) + await Promise.resolve() + await Promise.resolve() + expect(unhandled).toEqual([]) + } finally { + process.off('unhandledRejection', onUnhandled) + vi.useRealTimers() + } }) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index b72b4f4fe3..4a1c2b6bd3 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -4,6 +4,12 @@ import { mailboxRpc, type MailboxEnv, } from './mailbox-client.ts' +import { + recordMailboxParityEvent, + type MailboxMirrorOperation, + type MailboxMirrorOutcome, + type MailboxParityEnv, +} from './mailbox-parity-events.ts' import { toMailboxAttachmentInput, toMailboxMessageInput, @@ -31,15 +37,23 @@ import { * Best-effort D1 → Mailbox dual-write helpers. * * Never throw into D1 authority paths: catch/log with stable tags and return - * structured `mirrored | stale | skipped | error` outcomes. `system:email` - * stays in D1 only. + * structured outcomes (`mirrored | stale | missing | timeout | skipped | + * error`). `system:email` stays in D1 only. + * + * Every awaited DO RPC is bounded by {@link mailboxMirrorRpcTimeoutMs}. Partial + * mutation `missing` stays distinct from `mirrored`; delete `missing` maps to + * `mirrored` because desired absence is achieved. Outcomes are recorded via + * `recordMailboxParityEvent` when a user id is known. * * Delivery-event snapshots should be loaded via * `getMailboxDeliveryEventMirrorInput` (mailbox-snapshot-repo) so callers do * not supply promoted D1 columns individually. */ -export type MailboxMirrorEnv = MailboxEnv +/** Bound every awaited Mailbox DO mirror RPC (~1s). */ +export const mailboxMirrorRpcTimeoutMs = 1_000 + +export type MailboxMirrorEnv = MailboxEnv & MailboxParityEnv export type MailboxMirrorSkipReason = | 'system-email' @@ -49,6 +63,8 @@ export type MailboxMirrorSkipReason = export type MailboxMirrorResult = | { status: 'mirrored' } | { status: 'stale' } + | { status: 'missing' } + | { status: 'timeout' } | { status: 'skipped'; reason: MailboxMirrorSkipReason } | { status: 'error'; error: unknown } @@ -76,14 +92,49 @@ function skipIfMailboxMissing( return null } -/** Map touch/update/classify: accepted/missing → mirrored; stale → stale. */ +function mirrorOutcome(result: MailboxMirrorResult): MailboxMirrorOutcome { + switch (result.status) { + case 'mirrored': + case 'stale': + case 'missing': + case 'timeout': + case 'error': + return result.status + case 'skipped': + return 'skipped' + default: { + const exhaustive: never = result + throw new Error( + `Unhandled mailbox mirror result: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +function recordMirrorOutcome( + env: MailboxMirrorEnv, + userId: string | null | undefined, + operation: MailboxMirrorOperation, + result: MailboxMirrorResult, +): void { + if (userId == null || userId.length === 0) return + recordMailboxParityEvent(env, { + userId, + category: 'mirror', + operation, + outcome: mirrorOutcome(result), + }) +} + +/** Map touch/update/classify: accepted → mirrored; missing/stale stay distinct. */ function outcomeFromPartialMutation( result: MailboxPartialMutationResult, ): MailboxMirrorResult { switch (result.status) { case 'accepted': - case 'missing': return { status: 'mirrored' } + case 'missing': + return { status: 'missing' } case 'stale': return { status: 'stale' } default: { @@ -118,16 +169,66 @@ function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { return accepted ? { status: 'mirrored' } : { status: 'stale' } } -async function runMirror( - tag: string, - run: () => Promise, -): Promise { +/** + * Race a DO RPC against {@link mailboxMirrorRpcTimeoutMs}. + * The settled wrapper ensures a late rejection cannot become unhandled. + */ +export async function awaitMailboxMirrorRpc( + rpc: Promise, + timeoutMs: number = mailboxMirrorRpcTimeoutMs, +): Promise<{ ok: true; value: T } | { ok: false; timedOut: true }> { + const settled = rpc.then( + (value) => ({ kind: 'fulfilled' as const, value }), + (error: unknown) => ({ kind: 'rejected' as const, error }), + ) + let timeoutId: ReturnType | undefined + try { + const raced = await Promise.race([ + settled, + new Promise<{ kind: 'timeout' }>((resolve) => { + timeoutId = setTimeout(() => { + resolve({ kind: 'timeout' }) + }, timeoutMs) + }), + ]) + if (raced.kind === 'timeout') { + return { ok: false, timedOut: true } + } + if (raced.kind === 'rejected') { + throw raced.error + } + return { ok: true, value: raced.value } + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId) + } +} + +async function runMirror(input: { + tag: string + env: MailboxMirrorEnv + userId: string | null | undefined + operation: MailboxMirrorOperation + run: () => Promise +}): Promise { + let result: MailboxMirrorResult try { - return await run() + result = await input.run() } catch (error) { - console.warn(tag, error) - return { status: 'error', error } + console.warn(input.tag, error) + result = { status: 'error', error } } + recordMirrorOutcome(input.env, input.userId, input.operation, result) + return result +} + +function finishEarly(input: { + env: MailboxMirrorEnv + userId: string | null | undefined + operation: MailboxMirrorOperation + result: MailboxMirrorResult +}): MailboxMirrorResult { + recordMirrorOutcome(input.env, input.userId, input.operation, input.result) + return input.result } /** Best-effort full message (+ optional thread/attachments) snapshot mirror. */ @@ -137,22 +238,46 @@ export async function mirrorMailboxMessageSnapshot(input: { message: EmailMessageRecord attachments?: Array }): Promise { + const operation = 'mirror_message' as const const owner = resolveMirrorOwner(input.message.userId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.message.userId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-message-failed', async () => { - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).mirrorMessage({ - ownerId: owner.ownerId, - thread: input.thread ? toMailboxThreadInput(input.thread) : null, - message: toMailboxMessageInput(input.message), - attachments: input.attachments?.map(toMailboxAttachmentInput), + operation, + result: skippedMailbox, }) - return outcomeFromAccepted(result.accepted) + } + + return runMirror({ + tag: 'mailbox-mirror-message-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).mirrorMessage({ + ownerId: owner.ownerId, + thread: input.thread ? toMailboxThreadInput(input.thread) : null, + message: toMailboxMessageInput(input.message), + attachments: input.attachments?.map(toMailboxAttachmentInput), + }), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromAccepted(raced.value.accepted) + }, }) } @@ -173,21 +298,45 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { deliveryStatusAt: string } | null }): Promise { + const operation = 'upsert_delivery_event' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-delivery-event-failed', async () => { - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).upsertDeliveryEvent({ - ownerId: owner.ownerId, - event: input.event, - latestDeliveryStatus: input.latestDeliveryStatus, + operation, + result: skippedMailbox, }) - return outcomeFromAccepted(result.accepted) + } + + return runMirror({ + tag: 'mailbox-mirror-delivery-event-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).upsertDeliveryEvent({ + ownerId: owner.ownerId, + event: input.event, + latestDeliveryStatus: input.latestDeliveryStatus, + }), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromAccepted(raced.value.accepted) + }, }) } @@ -195,18 +344,42 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { export async function mirrorMailboxTouchThread( input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'touch_thread' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-touch-thread-failed', async () => { - const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).touchThread(rpcInput) - return outcomeFromPartialMutation(result) + operation, + result: skippedMailbox, + }) + } + + return runMirror({ + tag: 'mailbox-mirror-touch-thread-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const { env: _env, ...rpcInput } = input + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).touchThread(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromPartialMutation(raced.value) + }, }) } @@ -214,84 +387,171 @@ export async function mirrorMailboxTouchThread( export async function mirrorMailboxUpdateMessageDelivery( input: MailboxUpdateMessageDeliveryInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'update_message_delivery' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox + if (skippedMailbox) { + return finishEarly({ + env: input.env, + userId: owner.ownerId, + operation, + result: skippedMailbox, + }) + } - return runMirror( - 'mailbox-mirror-update-message-delivery-failed', - async () => { + return runMirror({ + tag: 'mailbox-mirror-update-message-delivery-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).updateMessageDelivery(rpcInput) - return outcomeFromPartialMutation(result) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).updateMessageDelivery(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromPartialMutation(raced.value) }, - ) + }) } /** Best-effort inbound classification update mirror. */ export async function mirrorMailboxSetMessageClassification( input: MailboxSetMessageClassificationInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'set_message_classification' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox + if (skippedMailbox) { + return finishEarly({ + env: input.env, + userId: owner.ownerId, + operation, + result: skippedMailbox, + }) + } - return runMirror( - 'mailbox-mirror-set-message-classification-failed', - async () => { + return runMirror({ + tag: 'mailbox-mirror-set-message-classification-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).setMessageClassification(rpcInput) - return outcomeFromPartialMutation(result) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).setMessageClassification(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromPartialMutation(raced.value) }, - ) + }) } /** Best-effort metadata-only message delete mirror (no R2). */ export async function mirrorMailboxDeleteMessageMetadata( input: MailboxDeleteMessageMetadataInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'delete_message_metadata' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox + if (skippedMailbox) { + return finishEarly({ + env: input.env, + userId: owner.ownerId, + operation, + result: skippedMailbox, + }) + } - return runMirror( - 'mailbox-mirror-delete-message-metadata-failed', - async () => { + return runMirror({ + tag: 'mailbox-mirror-delete-message-metadata-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteMessageMetadata(rpcInput) - return outcomeFromDeleteResult(result) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteMessageMetadata(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromDeleteResult(raced.value) }, - ) + }) } /** Best-effort delivery-event delete mirror. */ export async function mirrorMailboxDeleteDeliveryEvent( input: MailboxDeleteDeliveryEventInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'delete_delivery_event' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-delete-delivery-event-failed', async () => { - const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).deleteDeliveryEvent(rpcInput) - return outcomeFromDeleteResult(result) + operation, + result: skippedMailbox, + }) + } + + return runMirror({ + tag: 'mailbox-mirror-delete-delivery-event-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const { env: _env, ...rpcInput } = input + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteDeliveryEvent(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromDeleteResult(raced.value) + }, }) } @@ -299,17 +559,41 @@ export async function mirrorMailboxDeleteDeliveryEvent( export async function mirrorMailboxDeleteThreadIfEmpty( input: MailboxDeleteThreadIfEmptyInput & { env: MailboxMirrorEnv }, ): Promise { + const operation = 'delete_thread_if_empty' as const const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) return owner.result + if (!owner.ok) { + return finishEarly({ + env: input.env, + userId: input.ownerId, + operation, + result: owner.result, + }) + } const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) return skippedMailbox - - return runMirror('mailbox-mirror-delete-thread-if-empty-failed', async () => { - const { env: _env, ...rpcInput } = input - const result = await mailboxRpc({ + if (skippedMailbox) { + return finishEarly({ env: input.env, userId: owner.ownerId, - }).deleteThreadIfEmpty(rpcInput) - return outcomeFromDeleteResult(result) + operation, + result: skippedMailbox, + }) + } + + return runMirror({ + tag: 'mailbox-mirror-delete-thread-if-empty-failed', + env: input.env, + userId: owner.ownerId, + operation, + run: async () => { + const { env: _env, ...rpcInput } = input + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).deleteThreadIfEmpty(rpcInput), + ) + if (!raced.ok) return { status: 'timeout' } + return outcomeFromDeleteResult(raced.value) + }, }) } diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 93c755631c..70cebc6313 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -29,16 +29,30 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = userId: 'user-1', category: 'mirror', operation: 'set_message_classification', - outcome: 'skipped', + outcome: 'missing', timestamp: '2026-07-30T10:06:00.000Z', }) recordMailboxParityEvent(env, { userId: 'user-1', category: 'mirror', operation: 'delete_message_metadata', - outcome: 'error', + outcome: 'timeout', timestamp: '2026-07-30T10:07:00.000Z', }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'touch_thread', + outcome: 'skipped', + timestamp: '2026-07-30T10:08:00.000Z', + }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'upsert_delivery_event', + outcome: 'error', + timestamp: '2026-07-30T10:09:00.000Z', + }) recordMailboxParityEvent(env, { userId: 'user-2', category: 'parity', @@ -56,7 +70,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = timestamp: '2026-07-30T12:00:00.000Z', }) - expect(writeDataPoint).toHaveBeenCalledTimes(6) + expect(writeDataPoint).toHaveBeenCalledTimes(8) expect(writeDataPoint).toHaveBeenNthCalledWith(1, { indexes: ['user-1'], blobs: [ @@ -75,7 +89,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = indexes: ['user-1'], blobs: [ 'mailbox_mirror:set_message_classification', - 'skipped', + 'missing', '2026-07-30T10:06:00.000Z', ], doubles: [1, 0], @@ -84,12 +98,30 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = indexes: ['user-1'], blobs: [ 'mailbox_mirror:delete_message_metadata', - 'error', + 'timeout', '2026-07-30T10:07:00.000Z', ], doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(5, { + indexes: ['user-1'], + blobs: [ + 'mailbox_mirror:touch_thread', + 'skipped', + '2026-07-30T10:08:00.000Z', + ], + doubles: [1, 0], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(6, { + indexes: ['user-1'], + blobs: [ + 'mailbox_mirror:upsert_delivery_event', + 'error', + '2026-07-30T10:09:00.000Z', + ], + doubles: [1, 0], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(7, { indexes: ['user-2'], blobs: [ 'mailbox_parity:compare_messages', @@ -98,7 +130,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = ], doubles: [1, 2], }) - expect(writeDataPoint).toHaveBeenNthCalledWith(6, { + expect(writeDataPoint).toHaveBeenNthCalledWith(8, { indexes: ['user-3'], blobs: [ 'mailbox_parity:compare_threads', diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts index 66f394eee9..bb2c564770 100644 --- a/packages/worker/src/email/mailbox-parity-events.ts +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -14,7 +14,13 @@ export type MailboxMirrorOperation = | 'delete_delivery_event' | 'delete_thread_if_empty' -export type MailboxMirrorOutcome = 'mirrored' | 'stale' | 'skipped' | 'error' +export type MailboxMirrorOutcome = + | 'mirrored' + | 'stale' + | 'missing' + | 'timeout' + | 'skipped' + | 'error' export type MailboxParityOperation = | 'compare_threads' diff --git a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts index fb55632566..2357861335 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts @@ -4,6 +4,8 @@ import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.t import { getEmailDeliveryEventMirrorProjection, getMailboxDeliveryEventMirrorInput, + listEmailDeliveryEventMirrorProjectionsForMessage, + listMailboxDeliveryEventMirrorInputsForMessage, } from './mailbox-snapshot-repo.ts' import { ensureEmailTestSchema } from './test-schema.ts' @@ -117,4 +119,57 @@ test('mailbox-snapshot-repo loads complete delivery-event projection and ready M updatedAt: '2026-07-02T10:00:00.000Z', }), ) + + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, usage_effect_recorded_at, usage_month, + usage_bytes, usage_duration_ms, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + 'evt-2', + 'msg-1', + 'user-aaa', + 'inbox-1', + 'sent', + 'kody', + null, + null, + '{}', + 0, + null, + null, + null, + null, + '2026-07-02T10:05:00.000Z', + ) + .run() + + const projections = await listEmailDeliveryEventMirrorProjectionsForMessage({ + db, + ownerId: 'user-aaa', + messageId: 'msg-1', + limit: 10, + }) + expect(projections.map((row) => row.id)).toEqual(['evt-1', 'evt-2']) + + const mailboxInputs = await listMailboxDeliveryEventMirrorInputsForMessage({ + db, + ownerId: 'user-aaa', + messageId: 'msg-1', + limit: 10, + }) + expect(mailboxInputs).toEqual([ + expect.objectContaining({ + id: 'evt-1', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + expect.objectContaining({ + id: 'evt-2', + updatedAt: '2026-07-02T10:05:00.000Z', + }), + ]) }) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index 5fafe8ee09..9d732c35a3 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -35,6 +35,24 @@ type DeliveryEventMirrorRow = { created_at: string } +const deliveryEventMirrorSelect = `SELECT + id, + message_id, + user_id, + inbox_id, + event_type, + provider, + provider_message_id, + provider_event_id, + detail_json, + needs_effect_reconcile, + usage_effect_recorded_at, + usage_month, + usage_bytes, + usage_duration_ms, + created_at + FROM email_delivery_events` + function mapDeliveryEventMirrorRow( row: DeliveryEventMirrorRow, ): EmailDeliveryEventMirrorProjection { @@ -74,23 +92,7 @@ export async function getEmailDeliveryEventMirrorProjection(input: { }): Promise { const row = await input.db .prepare( - `SELECT - id, - message_id, - user_id, - inbox_id, - event_type, - provider, - provider_message_id, - provider_event_id, - detail_json, - needs_effect_reconcile, - usage_effect_recorded_at, - usage_month, - usage_bytes, - usage_duration_ms, - created_at - FROM email_delivery_events + `${deliveryEventMirrorSelect} WHERE id = ? AND user_id = ? LIMIT 1`, @@ -124,3 +126,46 @@ export async function getMailboxDeliveryEventMirrorInput(input: { sourceMutationAt: input.sourceMutationAt, }) } + +/** + * Load complete delivery-event mirror projections for one message, oldest + * first, bounded by `limit`. Scoped by owner. + */ +export async function listEmailDeliveryEventMirrorProjectionsForMessage(input: { + db: D1Database + ownerId: string + messageId: string + limit: number +}): Promise> { + const result = await input.db + .prepare( + `${deliveryEventMirrorSelect} + WHERE message_id = ? + AND user_id = ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.messageId, input.ownerId, input.limit) + .all() + return (result.results ?? []).map(mapDeliveryEventMirrorRow) +} + +/** + * Load ready Mailbox delivery-event snapshots for one message. + * Uses each event's `created_at` as `sourceMutationAt` (immutable inserts). + */ +export async function listMailboxDeliveryEventMirrorInputsForMessage(input: { + db: D1Database + ownerId: string + messageId: string + limit: number +}): Promise> { + const projections = + await listEmailDeliveryEventMirrorProjectionsForMessage(input) + return projections.map((projection) => + toMailboxDeliveryEventInput({ + projection, + sourceMutationAt: projection.createdAt, + }), + ) +} diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index b7a73592f8..ea135691fc 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -17,6 +17,10 @@ import { } from '#worker/entitlements/service.ts' import { recordUsage } from '#worker/usage/record-usage.ts' import { normalizeEmailAddress } from './address.ts' +import { + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, +} from './mailbox-live-mirror.ts' import { emailOutboundPausedMessage } from './outbound-abuse.ts' import { resolveUserPlatformSender } from './platform-address.ts' import { @@ -34,7 +38,11 @@ import { insertEmailMessageWithoutRawMime, updateEmailMessageDelivery, } from './service.ts' -import { type EmailMessageRecord, type EmailProcessingStatus } from './types.ts' +import { + type EmailMessageRecord, + type EmailProcessingStatus, + type EmailThreadRecord, +} from './types.ts' type SendEmailEnv = Pick< Env, @@ -48,8 +56,11 @@ type SendEmailEnv = Pick< | 'CLOUDFLARE_API_BASE_URL' | 'CLOUDFLARE_API_TOKEN' | 'USER_METER' + | 'MAILBOX' + | 'EMAIL_EVENTS' > & - EmailReportingEnv + EmailReportingEnv & + MailboxLiveMirrorEnv /** * Ceiling on the number of files per outbound message. Total size is @@ -324,6 +335,27 @@ async function requireStoredEmailMessage(input: { return stored } +/** + * One bounded never-throw D1 → Mailbox graph mirror at outbound terminals. + * Passes a just-created thread when present; otherwise omits `thread` so + * `mirrorMailboxMessageGraphFromD1` loads it from D1. Failures never affect + * send/refund. + */ +async function mirrorOutboundMailboxMessageGraph(input: { + env: SendEmailEnv + userId: string + messageId: string + createdThread: EmailThreadRecord | null +}) { + await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + messageId: input.messageId, + ...(input.createdThread ? { thread: input.createdThread } : {}), + }) +} + async function sendViaBinding(input: { env: SendEmailEnv from: string @@ -571,7 +603,7 @@ export async function sendOutboundEmail( }) const existingThreadId = original?.threadId ?? input.threadId ?? null - const thread = existingThreadId + const createdThread = existingThreadId ? null : await createEmailThread({ db: input.env.APP_DB, @@ -581,7 +613,7 @@ export async function sendOutboundEmail( rootMessageIdHeader: input.inReplyToHeader ?? null, lastMessageAt: new Date().toISOString(), }) - const threadId = existingThreadId ?? thread?.id ?? null + const threadId = existingThreadId ?? createdThread?.id ?? null const providerHeaders = buildProviderHeaders(storedHeaders) const message = await insertEmailMessageWithoutRawMime({ db: input.env.APP_DB, @@ -617,6 +649,13 @@ export async function sendOutboundEmail( sentAt: null, }, }) + const mirrorMailboxGraph = () => + mirrorOutboundMailboxMessageGraph({ + env: input.env, + userId: input.userId, + messageId: message.id, + createdThread, + }) try { await storeOutboundAttachments({ db: input.env.APP_DB, @@ -655,6 +694,7 @@ export async function sendOutboundEmail( }).catch((eventError) => { console.warn('email-attachment-failure-event-insert-failed', eventError) }) + await mirrorMailboxGraph() throw error } await insertEmailDeliveryEvent({ @@ -722,6 +762,7 @@ export async function sendOutboundEmail( providerMessageId, detail: { providerMessageId }, }) + await mirrorMailboxGraph() return { message: await requireStoredEmailMessage({ env: input.env, @@ -757,6 +798,7 @@ export async function sendOutboundEmail( }).catch((eventError) => { console.warn('email-delivery-failure-event-insert-failed', eventError) }) + await mirrorMailboxGraph() return { message: (await getEmailMessageById({ diff --git a/packages/worker/src/email/outbound.workers.test.ts b/packages/worker/src/email/outbound.workers.test.ts index 3f87a19883..a1be15b7f7 100644 --- a/packages/worker/src/email/outbound.workers.test.ts +++ b/packages/worker/src/email/outbound.workers.test.ts @@ -6,6 +6,8 @@ import { bytesToBase64 } from '@kody-internal/shared/base64.ts' import { McpCallerError } from '#mcp/caller-error.ts' import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' import { ensureEmailTestSchema } from './test-schema.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import { rpcFor } from './mailbox-test-helpers.ts' import { deleteEmailMessageById, getEmailMessageById, @@ -48,6 +50,37 @@ function createBindingSendEnv() { } } +/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ +function createMailboxStubEnv(input: { + mode: 'throw' | 'hang' + base?: typeof env +}) { + const base = input.base ?? env + const hang = () => new Promise(() => {}) + const fail = async () => { + throw new Error('simulated mailbox failure') + } + const method = input.mode === 'hang' ? hang : fail + const stub = { + mirrorMessage: method, + upsertDeliveryEvent: method, + touchThread: method, + updateMessageDelivery: method, + setMessageClassification: method, + deleteMessageMetadata: method, + deleteDeliveryEvent: method, + deleteThreadIfEmpty: method, + } + return { + ...base, + APP_BASE_URL: platformBaseUrl, + MAILBOX: { + idFromName: (name: string) => base.MAILBOX.idFromName(name), + get: () => stub, + } as unknown as DurableObjectNamespace, + } +} + async function seedVerifiedAccount(input: { email: string username?: string @@ -966,3 +999,212 @@ test('sendOutboundEmail caps max-plan users at the email daily backstop', async maxPlanEmailLimits.email_sends_per_day, ) }, 30_000) + +test('sendOutboundEmail mirrors outbound message graph into Mailbox on success', async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-sent-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]) + const sendEnv = { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-sent' } + }, + }, + } + + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror sent', + text: 'Mirrored body', + attachments: [ + { + filename: 'note.pdf', + contentType: 'application/pdf', + contentBase64: bytesToBase64(pdfBytes), + }, + ], + }) + + expect(result.status).toBe('sent') + expect(result.message.threadId).toBeTruthy() + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'sent', + providerMessageId: 'provider-mailbox-sent', + subject: 'Mailbox mirror sent', + threadId: result.message.threadId, + }) + const mirroredThread = await mailbox.getThread({ + threadId: result.message.threadId!, + }) + expect(mirroredThread).toMatchObject({ + id: result.message.threadId, + subjectNormalized: 'mailbox mirror sent', + }) + const mirroredAttachments = await mailbox.listAttachmentsForMessage({ + messageId: result.message.id, + }) + expect(mirroredAttachments).toEqual([ + expect.objectContaining({ + filename: 'note.pdf', + contentType: 'application/pdf', + size: pdfBytes.byteLength, + storageKind: 'external', + }), + ]) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'send_requested', + 'sent', + ]) +}, 30_000) + +test('sendOutboundEmail mirrors failed outbound graph into Mailbox', async () => { + silenceIncidentalRuntimeWarnings(['cloudflare-email-api-failed']) + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-fail-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + + using _server = createMswNodeServer( + [ + http.post(cloudflareEmailApi, () => + HttpResponse.json( + { success: false, errors: [{ message: 'provider down' }] }, + { status: 500 }, + ), + ), + ], + { onUnhandledRequest: 'bypass' }, + ) + + const result = await sendOutboundEmail({ + env: { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: undefined as unknown as SendEmail, + CLOUDFLARE_ACCOUNT_ID: 'account-123', + CLOUDFLARE_API_BASE_URL: 'https://api.cloudflare.test', + CLOUDFLARE_API_TOKEN: 'token-123', + }, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror failed', + text: 'Body', + }) + + expect(result.status).toBe('failed') + expect(result.error).toBe('provider down') + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'failed', + error: 'provider down', + threadId: result.message.threadId, + }) + expect( + await mailbox.getThread({ threadId: result.message.threadId! }), + ).toMatchObject({ id: result.message.threadId }) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'failed', + 'send_requested', + ]) +}, 30_000) + +test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-mirror-message-failed', + 'mailbox-mirror-delivery-event-failed', + 'mailbox-live-mirror-message-graph-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + + { + const accountEmail = `mailbox-err-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'throw' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-err' } + }, + }, + } + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox throw ignored', + text: 'Body', + }) + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-err', + error: null, + }) + const stored = await getEmailMessageById({ + db: env.APP_DB, + userId, + messageId: result.message.id, + }) + expect(stored?.processingStatus).toBe('sent') + } + + { + const accountEmail = `mailbox-hang-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'hang' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-hang' } + }, + }, + } + const startedAt = Date.now() + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox hang ignored', + text: 'Body', + }) + const elapsedMs = Date.now() - startedAt + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-hang', + error: null, + }) + // Message + send_requested + sent each race the mirror RPC bound. + expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 6) + } +}, 30_000) diff --git a/packages/worker/src/email/repo-body-limits.workers.test.ts b/packages/worker/src/email/repo-body-limits.workers.test.ts index 610d3c3c6a..cdb015dac8 100644 --- a/packages/worker/src/email/repo-body-limits.workers.test.ts +++ b/packages/worker/src/email/repo-body-limits.workers.test.ts @@ -4,7 +4,13 @@ import { maxRestorableTextColumnBytes, utf8ByteLength, } from '@kody-internal/shared/backup-restore-safety.ts' -import { emailBodyTruncationNotice, insertEmailMessage } from './repo.ts' +import { + createEmailThread, + emailBodyTruncationNotice, + getEmailThreadById, + insertEmailDeliveryEvent, + insertEmailMessage, +} from './repo.ts' import { ensureEmailTestSchema } from './test-schema.ts' test('stored email bodies are bounded so D1 backups stay importable', async () => { @@ -32,3 +38,60 @@ test('stored email bodies are bounded so D1 backups stay importable', async () = maxRestorableTextColumnBytes, ) }) + +test('getEmailThreadById is owner-scoped and insertEmailDeliveryEvent returns the row', async () => { + await ensureEmailTestSchema(env.APP_DB) + const ownerId = `owner-${crypto.randomUUID()}` + const otherUserId = `other-${crypto.randomUUID()}` + const thread = await createEmailThread({ + db: env.APP_DB, + userId: ownerId, + subjectNormalized: 'hello thread', + rootMessageIdHeader: '', + }) + + expect( + await getEmailThreadById({ + db: env.APP_DB, + userId: ownerId, + threadId: thread.id, + }), + ).toMatchObject({ + id: thread.id, + userId: ownerId, + subjectNormalized: 'hello thread', + }) + expect( + await getEmailThreadById({ + db: env.APP_DB, + userId: otherUserId, + threadId: thread.id, + }), + ).toBeNull() + + const createdAt = '2026-07-15T12:00:00.000Z' + const inserted = await insertEmailDeliveryEvent({ + db: env.APP_DB, + messageId: null, + userId: ownerId, + inboxId: null, + eventType: 'send_requested', + provider: 'cloudflare-email', + detail: { subject: 'hello' }, + createdAt, + }) + expect(inserted).toMatchObject({ + id: expect.any(String), + userId: ownerId, + eventType: 'send_requested', + provider: 'cloudflare-email', + detailJson: JSON.stringify({ subject: 'hello' }), + createdAt, + }) + const row = await env.APP_DB.prepare( + `SELECT id, created_at FROM email_delivery_events WHERE id = ?`, + ) + .bind(inserted.id) + .first<{ id: string; created_at: string }>() + expect(row).toEqual({ id: inserted.id, created_at: createdAt }) +}) diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 3503580936..c7fc822a29 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -524,6 +524,29 @@ async function findThreadForMessage(input: { export const findEmailThreadForInboundMessage = findThreadForMessage +/** + * Owner-scoped thread lookup for D1 → Mailbox graph mirrors and other + * strictly user-bound readers. Returns null when absent or owned by another + * user — never use for inbound delivery fencing. + */ +export async function getEmailThreadById(input: { + db: D1Database + userId: string + threadId: string +}) { + const row = await input.db + .prepare( + `SELECT * + FROM email_threads + WHERE id = ? + AND user_id = ? + LIMIT 1`, + ) + .bind(input.threadId, input.userId) + .first>() + return row ? mapThreadRow(row) : null +} + export async function createEmailThread(input: { db: D1Database id?: string @@ -1198,7 +1221,22 @@ export async function insertEmailDeliveryEvent(input: { providerEventId?: string | null detail?: Record | null createdAt?: string -}) { +}): Promise { + const id = crypto.randomUUID() + const createdAt = input.createdAt ?? nowIso() + const detailJson = JSON.stringify(input.detail ?? {}) + const record: EmailDeliveryEventRecord = { + id, + messageId: input.messageId ?? null, + userId: input.userId ?? null, + inboxId: input.inboxId ?? null, + eventType: input.eventType, + provider: input.provider ?? null, + providerMessageId: input.providerMessageId ?? null, + providerEventId: input.providerEventId ?? null, + detailJson, + createdAt, + } await input.db .prepare( `INSERT INTO email_delivery_events ( @@ -1207,16 +1245,17 @@ export async function insertEmailDeliveryEvent(input: { ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .bind( - crypto.randomUUID(), - input.messageId ?? null, - input.userId ?? null, - input.inboxId ?? null, - input.eventType, - input.provider ?? null, - input.providerMessageId ?? null, - input.providerEventId ?? null, - JSON.stringify(input.detail ?? {}), - input.createdAt ?? nowIso(), + record.id, + record.messageId, + record.userId, + record.inboxId, + record.eventType, + record.provider, + record.providerMessageId, + record.providerEventId, + record.detailJson, + record.createdAt, ) .run() + return record } diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts index 19979cd0c2..35f2299626 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts @@ -3,12 +3,21 @@ import { createMcpCallerContext } from '#mcp/context.ts' const mocks = vi.hoisted(() => ({ setEmailMessageClassification: vi.fn(), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'message-1', + message: { status: 'mirrored' }, + events: [], + })), })) vi.mock('#worker/email/repo.ts', () => ({ setEmailMessageClassification: mocks.setEmailMessageClassification, })) +vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, +})) + const { emailMessageClassifyCapability } = await import('./email-message-classify.ts') @@ -56,8 +65,15 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'quarantined', classificationReason: 'Reclassified by user.', }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: context.env, + db: context.env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + }) mocks.setEmailMessageClassification.mockResolvedValueOnce(true) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'message-1', classification: 'accepted' }, @@ -74,12 +90,39 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'accepted', classificationReason: null, }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: context.env, + db: context.env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + }) mocks.setEmailMessageClassification.mockResolvedValueOnce(false) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'missing', classification: 'quarantined' }, context, ), ).rejects.toThrow('Email message not found: missing') + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + + mocks.setEmailMessageClassification.mockResolvedValueOnce(true) + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'message-1', + message: { + status: 'error', + error: new Error('mailbox mirror timed out'), + }, + events: [], + }) + await expect( + emailMessageClassifyCapability.handler( + { message_id: 'message-1', classification: 'quarantined' }, + context, + ), + ).resolves.toEqual({ + message_id: 'message-1', + classification: 'quarantined', + }) }) diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts index ea10869745..1a9562ddf9 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts @@ -1,6 +1,7 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' +import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' import { setEmailMessageClassification } from '#worker/email/repo.ts' import { emailClassificationValues } from '#worker/email/types.ts' import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' @@ -44,6 +45,12 @@ export const emailMessageClassifyCapability = defineDomainCapability( if (!updated) { throw new Error(`Email message not found: ${args.message_id}`) } + await mirrorMailboxMessageGraphFromD1({ + env: ctx.env, + db: ctx.env.APP_DB, + userId: user.userId, + messageId: args.message_id, + }) return { message_id: args.message_id, classification: args.classification, From e9e9f184968909212005b77708dd9e90faedc0c7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 10:44:13 +0000 Subject: [PATCH 08/62] fix(email): batch Mailbox graph event mirrors Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 123 +++-- .../app/handlers/account-email.node.test.ts | 57 +-- .../worker/src/app/handlers/account-email.ts | 11 +- packages/worker/src/email/mailbox-do.ts | 39 ++ .../src/email/mailbox-do.workers.test.ts | 158 ++++++ .../email/mailbox-live-mirror.node.test.ts | 241 ++++++--- .../worker/src/email/mailbox-live-mirror.ts | 61 +-- .../src/email/mailbox-mirror.node.test.ts | 157 ++++++ packages/worker/src/email/mailbox-mirror.ts | 470 +++++++----------- .../email/mailbox-parity-events.node.test.ts | 20 +- .../worker/src/email/mailbox-parity-events.ts | 1 + .../email/mailbox-snapshot-repo.node.test.ts | 9 + .../worker/src/email/mailbox-snapshot-repo.ts | 12 +- packages/worker/src/email/mailbox-types.ts | 22 + .../outbound-mailbox-mirror.workers.test.ts | 277 +++++++++++ .../worker/src/email/outbound.workers.test.ts | 242 --------- packages/worker/src/email/repo.ts | 6 +- .../email/service-classification.node.test.ts | 119 +++++ packages/worker/src/email/service.ts | 39 ++ .../email/email-message-classify.node.test.ts | 47 +- .../email/email-message-classify.ts | 10 +- 21 files changed, 1318 insertions(+), 803 deletions(-) create mode 100644 packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts create mode 100644 packages/worker/src/email/service-classification.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 492f2bdb56..3ec77dc79e 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -220,9 +220,10 @@ Durable Object export behavior: RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires non-inbound live dual-write for outbound terminals, provider-recorded delivery - events, and user classification without changing D1 authority — D1 `email_*` - rows remain the live source of truth and are still exported in the `d1` - section. See [Mailbox](#durable-objects-mailbox). + events, and user classification (`service.ts#setEmailMessageClassification`) + without changing D1 authority — D1 `email_*` rows remain the live source of + truth and are still exported in the `d1` section. See + [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -394,11 +395,12 @@ do not feed the admin insights charts today: - `double2` — D1-minus-DO count delta (`d1Count - doCount` for parity compares; `0` when absent or not applicable) -Mirror operations: `mirror_message`, `upsert_delivery_event`, `touch_thread`, -`update_message_delivery`, `set_message_classification`, -`delete_message_metadata`, `delete_delivery_event`, `delete_thread_if_empty`. -Mirror outcomes: `mirrored`, `stale`, `missing`, `timeout`, `skipped`, `error`. -Parity operations: `compare_threads`, `compare_messages`, `compare_attachments`, +Mirror operations: `mirror_message`, `upsert_delivery_event`, +`upsert_delivery_event_batch`, `touch_thread`, `update_message_delivery`, +`set_message_classification`, `delete_message_metadata`, +`delete_delivery_event`, `delete_thread_if_empty`. Mirror outcomes: `mirrored`, +`stale`, `missing`, `timeout`, `skipped`, `error`. Parity operations: +`compare_threads`, `compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Mirror outcome writes are automatic from live dual-write helpers; parity compare writers remain for future reconciliation. Writes are best-effort and never throw into D1 @@ -675,15 +677,17 @@ owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` - latest per-message delivery status on `email_messages.delivery_status`, kept separate from send-request `processing_status` -**Mirror write contract:** `mirrorMessage` and `upsertDeliveryEvent` take -complete snapshots — every persisted field is explicit (nullable fields use -explicit `null`). They are not patch APIs. Both require `ownerId` and apply -equal-or-newer `updatedAt` snapshots (stale snapshots are ignored, not applied). -The only omission exception is the `mirrorMessage` `attachments` bundle: -omitting it preserves existing attachment rows; an explicit `attachments: []` -clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and -external attachment `storageKey` values against the canonical builders for that -`ownerId`. +**Mirror write contract:** `mirrorMessage`, `upsertDeliveryEvent`, and +`upsertDeliveryEvents` take complete snapshots — every persisted field is +explicit (nullable fields use explicit `null`). They are not patch APIs. All +require `ownerId` and apply equal-or-newer `updatedAt` snapshots (stale +snapshots are ignored, not applied). `upsertDeliveryEvents` accepts at most +`mailboxUpsertDeliveryEventsMax` (100) events in one owner-bound batch RPC and +inserts them in caller order (chronological when loaded from D1). The only +omission exception is the `mirrorMessage` `attachments` bundle: omitting it +preserves existing attachment rows; an explicit `attachments: []` clears them. +Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment +`storageKey` values against the canonical builders for that `ownerId`. **Partial mutation RPCs** (library-only in the non-inbound phase-2 slice; owner-bound, monotonic on `updatedAt`, no R2 or retention-alarm side effects — @@ -735,13 +739,17 @@ bounded by `mailboxMirrorRpcTimeoutMs` (1 second). Each returns a structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, `{ status: 'stale' }`, `{ status: 'missing' }`, `{ status: 'timeout' }`, `{ status: 'skipped', reason }` (`system-email` | `mailbox-unconfigured` | -`missing-owner`), or `{ status: 'error', error }`. Every outcome is recorded -automatically via `recordMailboxParityEvent` (`mailbox_mirror:`) when -a user id is known; `system:email` is excluded. Failures log with stable tags -(for example `mailbox-mirror-message-failed`) and never propagate into D1 commit -paths. Helpers cover full snapshots (`mirrorMailboxMessageSnapshot`, -`mirrorMailboxDeliveryEventSnapshot` — prefer loading delivery events with -`getMailboxDeliveryEventMirrorInput`) and partial mutations +`missing-owner`), or `{ status: 'error', error }`. Single-RPC helpers record one +`mailbox_mirror:` outcome automatically when a user id is known; +`mirrorMailboxDeliveryEventSnapshots` records one +`mailbox_mirror:upsert_delivery_event_batch` outcome per batch (timeout/error +apply uniformly to per-event summary entries). `system:email` is excluded. +Failures log with stable tags (for example `mailbox-mirror-message-failed`) and +never propagate into D1 commit paths. Helpers cover full snapshots +(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot`, +`mirrorMailboxDeliveryEventSnapshots` — prefer loading delivery events with +`getMailboxDeliveryEventMirrorInput` or +`listMailboxDeliveryEventMirrorInputsForMessage`) and partial mutations (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). Partial @@ -751,13 +759,16 @@ below. D1 remains sole authority for all live mail read/write paths. **Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 message graph (optional caller thread, message, attachments, then delivery events) and mirrors it best-effort. `mirrorMailboxMessageGraphFromD1` settles -the message snapshot RPC first, then fans out delivery-event snapshot RPCs -concurrently so event-phase wall time stays near one RPC timeout (not limit × -timeout). Event load uses `max+1` rows so `eventsTruncated` is explicit; at most -`mailboxLiveMirrorMaxEvents` (`mailboxDefaultPageSize`, 100) events are mirrored -per call to stay under Analytics Engine's ~250 writes/request budget (1 message -outcome + up to 100 event outcomes). Never throws; returns a bounded summary. -**Live callers (non-inbound phase-2 slice):** +the message snapshot RPC first, then repairs delivery events with one +owner-bound `upsertDeliveryEvents` batch RPC (not concurrent per-event RPCs to +the same DO). Event load queries newest `max+1` rows from D1, restores +chronological order, and when truncated keeps the newest +`mailboxLiveMirrorMaxEvents` (`mailboxUpsertDeliveryEventsMax`, 100) — dropping +oldest overflow. The batch RPC shares one 1s timeout and one +`mailbox_mirror:upsert_delivery_event_batch` telemetry outcome (timeout/error +apply uniformly to per-event summary entries). Analytics Engine budget per graph +call is two writes (1 message outcome + 1 batch outcome). Never throws; returns +a bounded summary. **Live callers (non-inbound phase-2 slice):** - **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full @@ -767,10 +778,11 @@ outcome + up to 100 event outcomes). Never throws; returns a bounded summary. - **Provider-recorded delivery events** (`delivery-queue.ts`) — when queue ingestion persists a new provider event (`recorded`), schedules `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack is not - blocked; full graph repair, not a partial delivery-event touch. -- **User classification** (`account-email.ts`, `email-message-classify.ts`) — - after D1 classification update, mirrors the full graph (not the partial - `setMessageClassification` RPC alone). + blocked; full graph repair via message + batch event upsert. +- **User classification** (`service.ts#setEmailMessageClassification`) — + transport handlers (`account-email.ts`, `email-message-classify.ts`) delegate + here for the D1 mutation + full graph mirror invariant (mirror only after a + successful D1 update; failures never change the mutation response). Inbound commit/touch/classify paths, retention deletes, and partial mutation mirrors are **not** wired in this slice. Parity compare/reconcile cron remains @@ -800,23 +812,24 @@ This is an expand/contract migration. **Phase 2 is partially wired (non-inbound slice):** live dual-write covers outbound terminal message/thread/attachment/event graphs, provider-recorded delivery events (queue `waitUntil` full graph repair), and user classification (full graph repair after -D1 update). Mirror helpers emit automatic namespaced outcome telemetry -(`missing` and `timeout` included). Every awaited mirror RPC is bounded to 1s; -graph event fan-out is concurrent with explicit truncation when D1 has more than -`mailboxLiveMirrorMaxEvents` delivery events. **`system:email` stays in D1 -only.** Inbound lifecycle dual-write, retention/metadata deletes, and scheduled -parity reconcile are **not** wired in this slice. Live email read/write -authority is unchanged — D1 `email_*` tables remain authoritative for all live -paths. D1 email rows and existing R2 inventory deletion stay authoritative -during expand. **Phase 2 completes only when every user-mail mutation that -writes D1 also writes the DO and parity/reconciliation are wired** — inbound -paths and parity reconcile remain. +D1 update via `service.ts#setEmailMessageClassification`). Mirror helpers emit +automatic namespaced outcome telemetry (`missing` and `timeout` included). +Message and batch event repair each use one 1s-bounded RPC; graph event repair +batches up to 100 events with explicit truncation (newest retained, +chronological insertion restored) and one batch telemetry outcome. +**`system:email` stays in D1 only.** Inbound lifecycle dual-write, +retention/metadata deletes, and scheduled parity reconcile are **not** wired in +this slice. Live email read/write authority is unchanged — D1 `email_*` tables +remain authoritative for all live paths. D1 email rows and existing R2 inventory +deletion stay authoritative during expand. **Phase 2 completes only when every +user-mail mutation that writes D1 also writes the DO and parity/reconciliation +are wired** — inbound paths and parity reconcile remain. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / - `exportMailbox` / `countMailbox` / `purge` RPCs and alarm retention, and - register account deletion/export consumption. No dual-write; D1 remains sole - authority for live mail. + `upsertDeliveryEvents` / `exportMailbox` / `countMailbox` / `purge` RPCs and + alarm retention, and register account deletion/export consumption. No + dual-write; D1 remains sole authority for live mail. 2. **D1-authoritative dual-write + parity** — every user-mail mutation that writes D1 also writes the DO; parity counters and reconciliation prove DO completeness against D1. D1 stays authoritative for reads. Library surface: @@ -824,13 +837,15 @@ paths and parity reconcile remain. `mailbox-snapshot-repo.ts`, `mailbox-mirror.ts`, `mailbox-live-mirror.ts`, `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, - `deleteDeliveryEvent`, `deleteThreadIfEmpty`), and + `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, - provider `recorded` queue events (`waitUntil`), and user classification call + provider `recorded` queue events (`waitUntil`), and user classification + (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` - outcomes automatically. **Still pending for phase-2 completion:** inbound - commit/touch/classify dual-write, retention and metadata-delete mirrors, and - scheduled parity compare/reconcile. + outcomes automatically (graph repair uses message RPC + one batch event RPC). + **Still pending for phase-2 completion:** inbound commit/touch/classify + dual-write, retention and metadata-delete mirrors, and scheduled parity + compare/reconcile. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata diff --git a/packages/worker/src/app/handlers/account-email.node.test.ts b/packages/worker/src/app/handlers/account-email.node.test.ts index e206514d67..69293c4201 100644 --- a/packages/worker/src/app/handlers/account-email.node.test.ts +++ b/packages/worker/src/app/handlers/account-email.node.test.ts @@ -155,11 +155,6 @@ const mockModule = vi.hoisted(() => ({ }, ]), setEmailMessageClassification: vi.fn(async () => true), - mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ - messageId: 'msg-1', - message: { status: 'mirrored' }, - events: [], - })), prepare: vi.fn(), })) @@ -227,13 +222,11 @@ vi.mock('#worker/email/repo.ts', () => ({ mockModule.listEmailAttachmentsForMessage(...args), listEmailDeliveryEvents: (...args: Array) => mockModule.listEmailDeliveryEvents(...args), - setEmailMessageClassification: (...args: Array) => - mockModule.setEmailMessageClassification(...args), })) -vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: (...args: Array) => - mockModule.mirrorMailboxMessageGraphFromD1(...args), +vi.mock('#worker/email/service.ts', () => ({ + setEmailMessageClassification: (...args: Array) => + mockModule.setEmailMessageClassification(...args), })) const { createAccountEmailApiHandler } = await import('./account-email.ts') @@ -568,7 +561,6 @@ test('email API lists classification filters and classifies inbound messages', a }) mockModule.setEmailMessageClassification.mockClear() - mockModule.mirrorMailboxMessageGraphFromD1.mockClear() // Reload after classify uses the default message list again. createEnv({ meter, messageRows: [messageRow] }) const quarantineResponse = await handler.handler({ @@ -584,25 +576,19 @@ test('email API lists classification filters and classifies inbound messages', a }) expect(quarantineResponse.status).toBe(200) expect(mockModule.setEmailMessageClassification).toHaveBeenCalledWith({ + env, db: env.APP_DB, userId: 'stable-user-1', messageId: 'msg-1', classification: 'quarantined', classificationReason: 'Reclassified by user.', }) - expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env, - db: env.APP_DB, - userId: 'stable-user-1', - messageId: 'msg-1', - }) await expect(quarantineResponse.json()).resolves.toMatchObject({ ok: true, selectedMessage: expect.objectContaining({ id: 'msg-1' }), }) mockModule.setEmailMessageClassification.mockClear() - mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const acceptResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -616,21 +602,15 @@ test('email API lists classification filters and classifies inbound messages', a }) expect(acceptResponse.status).toBe(200) expect(mockModule.setEmailMessageClassification).toHaveBeenCalledWith({ + env, db: env.APP_DB, userId: 'stable-user-1', messageId: 'msg-1', classification: 'accepted', classificationReason: null, }) - expect(mockModule.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env, - db: env.APP_DB, - userId: 'stable-user-1', - messageId: 'msg-1', - }) mockModule.setEmailMessageClassification.mockResolvedValueOnce(false) - mockModule.mirrorMailboxMessageGraphFromD1.mockClear() const missingResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -643,33 +623,6 @@ test('email API lists classification filters and classifies inbound messages', a }), }) expect(missingResponse.status).toBe(404) - expect(mockModule.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() - - mockModule.setEmailMessageClassification.mockResolvedValueOnce(true) - mockModule.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ - messageId: 'msg-1', - message: { - status: 'error', - error: new Error('mailbox mirror timed out'), - }, - events: [], - }) - const mirrorFailureResponse = await handler.handler({ - request: new Request('https://example.com/account/email.json', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - action: 'classify', - message_id: 'msg-1', - classification: 'quarantined', - }), - }), - }) - expect(mirrorFailureResponse.status).toBe(200) - await expect(mirrorFailureResponse.json()).resolves.toMatchObject({ - ok: true, - selectedMessage: expect.objectContaining({ id: 'msg-1' }), - }) const invalidActionResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { diff --git a/packages/worker/src/app/handlers/account-email.ts b/packages/worker/src/app/handlers/account-email.ts index 62b8148201..e1e63b4551 100644 --- a/packages/worker/src/app/handlers/account-email.ts +++ b/packages/worker/src/app/handlers/account-email.ts @@ -7,8 +7,7 @@ import { requireAuthenticatedPageUser } from '#app/page-auth.ts' import { readTrimmedStringOrEmpty } from '#app/request-body.ts' import { type routes } from '#app/routes.ts' import { renderAppPage } from '#app/ssr-render.tsx' -import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' -import { setEmailMessageClassification } from '#worker/email/repo.ts' +import { setEmailMessageClassification } from '#worker/email/service.ts' import { emailClassificationValues, type EmailClassification, @@ -113,6 +112,7 @@ export function createAccountEmailApiHandler(env: Env) { } const updated = await setEmailMessageClassification({ + env, db: env.APP_DB, userId: user.mcpUser.userId, messageId, @@ -130,13 +130,6 @@ export function createAccountEmailApiHandler(env: Env) { ) } - await mirrorMailboxMessageGraphFromD1({ - env, - db: env.APP_DB, - userId: user.mcpUser.userId, - messageId, - }) - const listUrl = new URL(request.url, 'http://localhost') listUrl.searchParams.set('selected', messageId) return jsonResponse( diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index c8b06ce2f4..750ba38f96 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -23,6 +23,7 @@ import { } from './mailbox-mutations.ts' import { assertMailboxNonEmptyString, + mailboxUpsertDeliveryEventsMax, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, @@ -45,6 +46,8 @@ import { type MailboxThreadRecord, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, + type MailboxUpsertDeliveryEventBatchItemResult, + type MailboxUpsertDeliveryEventsResult, } from './mailbox-types.ts' /** @@ -232,6 +235,42 @@ class MailboxBase extends DurableObject implements MailboxRpc { return { inserted, accepted, updatedLatestStatus } } + /** + * Upsert a bounded batch of complete immutable delivery-event snapshots in + * one transaction / one DO RPC. Validates non-empty and + * {@link mailboxUpsertDeliveryEventsMax}. Does not patch message latest + * delivery status. Marks retention dirty once for the batch. + */ + async upsertDeliveryEvents(input: { + ownerId: string + events: Array + }): Promise { + if (!Array.isArray(input.events) || input.events.length === 0) { + throw new Error('Mailbox upsertDeliveryEvents events must be non-empty.') + } + if (input.events.length > mailboxUpsertDeliveryEventsMax) { + throw new Error( + `Mailbox upsertDeliveryEvents events exceed max of ${mailboxUpsertDeliveryEventsMax}.`, + ) + } + const results: Array = [] + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + for (const event of input.events) { + const eventId = assertMailboxNonEmptyString(event.id, 'event.id') + const write = this.store.writeDeliveryEventRow(event) + results.push({ + eventId, + inserted: write.inserted, + accepted: write.accepted, + }) + } + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return { results } + } + /** * Advance thread activity without a full snapshot. Equal/newer `updatedAt` * only; `last_message_at` never moves backward. diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index c619df9701..b1f5aa4c10 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -14,6 +14,7 @@ import { stubFor, uniqueUserId, } from './mailbox-test-helpers.ts' +import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', async () => { silenceIncidentalRuntimeWarnings() @@ -697,3 +698,160 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso deliveryEvents: 0, }) }) + +test('Mailbox upsertDeliveryEvents validates bounds, owns batch, and arms retention once', async () => { + silenceIncidentalRuntimeWarnings() + const userId = uniqueUserId('batch') + const otherUserId = uniqueUserId('batch-other') + const mailbox = rpcFor(userId) + const stub = stubFor(userId) + + const message = baseMessage(userId, { + id: 'batch-msg', + direction: 'outbound', + processingStatus: 'sent', + rawMimeKey: null, + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + await mailbox.mirrorMessage({ ownerId: userId, message }) + + await assertMailboxThrows(/events must be non-empty/, () => + mailbox.upsertDeliveryEvents({ ownerId: userId, events: [] }), + ) + const tooMany = Array.from( + { length: mailboxUpsertDeliveryEventsMax + 1 }, + (_, index) => + baseDeliveryEvent({ + id: `overflow-${index}`, + messageId: message.id, + eventType: 'sent', + provider: 'kody', + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + }), + ) + await assertMailboxThrows(/exceed max of/, () => + mailbox.upsertDeliveryEvents({ ownerId: userId, events: tooMany }), + ) + + await runInDurableObject(stub, async (_instance: Mailbox, state) => { + await state.storage.deleteAlarm() + }) + + const batch = [ + baseDeliveryEvent({ + id: 'batch-evt-1', + messageId: message.id, + eventType: 'send_requested', + provider: 'kody', + createdAt: '2026-07-02T10:00:01.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }), + baseDeliveryEvent({ + id: 'batch-evt-2', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + providerEventId: 'provider-batch-2', + createdAt: '2026-07-02T10:00:02.000Z', + updatedAt: '2026-07-02T10:00:02.000Z', + }), + ] + const first = await mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: batch, + }) + expect(first).toEqual({ + results: [ + { eventId: 'batch-evt-1', inserted: true, accepted: true }, + { eventId: 'batch-evt-2', inserted: true, accepted: true }, + ], + }) + // Batch must not patch message latest delivery status. + expect(await mailbox.getMessage({ messageId: message.id })).toMatchObject({ + deliveryStatus: 'delivered', + deliveryStatusAt: '2026-07-02T10:00:00.000Z', + }) + expect( + new Set( + ( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 10 }) + ).map((event) => event.id), + ), + ).toEqual(new Set(['batch-evt-1', 'batch-evt-2'])) + + const alarmAfterBatch = await runInDurableObject( + stub, + async (_instance: Mailbox, state) => state.storage.getAlarm(), + ) + expect(alarmAfterBatch).toBeTypeOf('number') + + const replay = await mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: [ + batch[0]!, + baseDeliveryEvent({ + id: 'batch-evt-2-dup', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + providerEventId: 'provider-batch-2', + createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T11:00:00.000Z', + }), + ], + }) + expect(replay).toEqual({ + results: [ + { eventId: 'batch-evt-1', inserted: false, accepted: true }, + { eventId: 'batch-evt-2-dup', inserted: false, accepted: false }, + ], + }) + + await assertMailboxThrows(/ownerId mismatch/, () => + mailbox.upsertDeliveryEvents({ + ownerId: otherUserId, + events: [ + baseDeliveryEvent({ + id: 'batch-evt-other', + messageId: message.id, + eventType: 'sent', + provider: 'kody', + }), + ], + }), + ) + + // Transactionality: a mid-batch validation failure rolls back prior writes. + const beforeCount = (await mailbox.countMailbox()).deliveryEvents + await assertMailboxThrows(/canonical ISO-8601/, () => + mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: [ + baseDeliveryEvent({ + id: 'batch-txn-ok', + messageId: message.id, + eventType: 'delivered', + provider: 'kody', + createdAt: '2026-07-02T12:00:00.000Z', + updatedAt: '2026-07-02T12:00:00.000Z', + }), + baseDeliveryEvent({ + id: 'batch-txn-bad', + messageId: message.id, + eventType: 'delivered', + provider: 'kody', + createdAt: '2026-07-02T12:00:00Z', + updatedAt: '2026-07-02T12:00:00Z', + }), + ], + }), + ) + expect((await mailbox.countMailbox()).deliveryEvents).toBe(beforeCount) + expect( + ( + await mailbox.listDeliveryEvents({ messageId: message.id, limit: 20 }) + ).some((event) => event.id === 'batch-txn-ok'), + ).toBe(false) +}) diff --git a/packages/worker/src/email/mailbox-live-mirror.node.test.ts b/packages/worker/src/email/mailbox-live-mirror.node.test.ts index af8ae5ba64..91be2c89f8 100644 --- a/packages/worker/src/email/mailbox-live-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-live-mirror.node.test.ts @@ -11,6 +11,7 @@ import { } from './mailbox-live-mirror.ts' import { ensureEmailTestSchema } from './test-schema.ts' import { type EmailThreadRecord } from './types.ts' +import { type MailboxDeliveryEventInput } from './mailbox-types.ts' async function createEmailDb() { const sqlite = new DatabaseSync(':memory:') @@ -137,12 +138,10 @@ async function seedMessageGraph(db: D1Database) { test('live-mirror AE write budget stays under Analytics Engine request cap', () => { expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) - expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe( - 1 + mailboxLiveMirrorMaxEvents, - ) + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe(2) }) -test('live-mirror graph mirrors message before events and uses createdAt for event updatedAt', async () => { +test('live-mirror graph mirrors message before one batch event RPC and uses createdAt for event updatedAt', async () => { const { db } = await createEmailDb() await seedMessageGraph(db) @@ -151,18 +150,28 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve messageSettled = true return { ok: true as const, accepted: true } }) - const upsertDeliveryEvent = vi.fn( - async (input: { event: { id: string } }) => { + const upsertDeliveryEvents = vi.fn( + async (input: { events: Array<{ id: string }> }) => { expect(messageSettled).toBe(true) return { - inserted: true, - accepted: true, - updatedLatestStatus: false, - eventId: input.event.id, + results: input.events.map((event) => ({ + eventId: event.id, + inserted: true, + accepted: true, + })), } }, ) - const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) + const upsertDeliveryEvent = vi.fn(async () => ({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + })) + const { env } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents, + upsertDeliveryEvent, + }) const thread: EmailThreadRecord = { id: 'thread-1', @@ -180,7 +189,6 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve db, userId: 'user-aaa', messageId: 'msg-1', - sourceMutationAt: '2026-07-01T12:00:03.000Z', thread, }) @@ -203,10 +211,12 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve ], }), ) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + expect(upsertDeliveryEvent).not.toHaveBeenCalled() // When thread is omitted, load it from D1 via getEmailThreadById. mirrorMessage.mockClear() - upsertDeliveryEvent.mockClear() + upsertDeliveryEvents.mockClear() await mirrorMailboxMessageGraphFromD1({ env, db, @@ -221,27 +231,26 @@ test('live-mirror graph mirrors message before events and uses createdAt for eve }), }), ) - expect(upsertDeliveryEvent).toHaveBeenCalledWith( - expect.objectContaining({ - event: expect.objectContaining({ - id: 'evt-1', - createdAt: '2026-07-01T12:00:01.000Z', - updatedAt: '2026-07-01T12:00:01.000Z', - }), - }), - ) - expect(upsertDeliveryEvent).toHaveBeenCalledWith( + expect(upsertDeliveryEvents).toHaveBeenCalledWith( expect.objectContaining({ - event: expect.objectContaining({ - id: 'evt-2', - createdAt: '2026-07-01T12:00:02.000Z', - updatedAt: '2026-07-01T12:00:02.000Z', - }), + ownerId: 'user-aaa', + events: [ + expect.objectContaining({ + id: 'evt-1', + createdAt: '2026-07-01T12:00:01.000Z', + updatedAt: '2026-07-01T12:00:01.000Z', + }), + expect.objectContaining({ + id: 'evt-2', + createdAt: '2026-07-01T12:00:02.000Z', + updatedAt: '2026-07-01T12:00:02.000Z', + }), + ], }), ) }) -test('live-mirror graph fans out event RPCs concurrently after message settles', async () => { +test('live-mirror graph uses one batch RPC after message settles, not per-event fan-out', async () => { vi.useFakeTimers() try { const { db } = await createEmailDb() @@ -255,25 +264,35 @@ test('live-mirror graph fans out event RPCs concurrently after message settles', }) } - const eventDelayMs = 400 - let maxInFlight = 0 - let inFlight = 0 + const batchDelayMs = 400 let messageSettled = false + let batchCalls = 0 const mirrorMessage = vi.fn(async () => { messageSettled = true return { ok: true as const, accepted: true } }) - const upsertDeliveryEvent = vi.fn(async () => { - expect(messageSettled).toBe(true) - inFlight += 1 - maxInFlight = Math.max(maxInFlight, inFlight) - await new Promise((resolve) => { - setTimeout(resolve, eventDelayMs) - }) - inFlight -= 1 - return { inserted: true, accepted: true, updatedLatestStatus: false } + const upsertDeliveryEvents = vi.fn( + async (input: { events: Array<{ id: string }> }) => { + expect(messageSettled).toBe(true) + batchCalls += 1 + await new Promise((resolve) => { + setTimeout(resolve, batchDelayMs) + }) + return { + results: input.events.map((event) => ({ + eventId: event.id, + inserted: true, + accepted: true, + })), + } + }, + ) + const upsertDeliveryEvent = vi.fn() + const { env } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents, + upsertDeliveryEvent, }) - const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvent }) const pending = mirrorMailboxMessageGraphFromD1({ env, @@ -282,22 +301,21 @@ test('live-mirror graph fans out event RPCs concurrently after message settles', messageId: 'msg-1', }) - // One event-delay tick completes the whole fan-out (not eventCount × delay). - await vi.advanceTimersByTimeAsync(eventDelayMs) + await vi.advanceTimersByTimeAsync(batchDelayMs) const summary = await pending expect(summary.events).toHaveLength(eventCount) expect(summary.eventsTruncated).toBe(false) - expect(maxInFlight).toBe(eventCount) - expect(upsertDeliveryEvent).toHaveBeenCalledTimes(eventCount) - // Wall budget stays near one RPC timeout for the event phase. - expect(eventDelayMs).toBeLessThan(mailboxMirrorRpcTimeoutMs) + expect(batchCalls).toBe(1) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + expect(upsertDeliveryEvent).not.toHaveBeenCalled() + expect(batchDelayMs).toBeLessThan(mailboxMirrorRpcTimeoutMs) } finally { vi.useRealTimers() } }) -test('live-mirror graph sets eventsTruncated and mirrors at most max events', async () => { +test('live-mirror graph keeps newest events when truncated and sets eventsTruncated', async () => { const { db } = await createEmailDb() await seedMessage(db) @@ -312,18 +330,22 @@ test('live-mirror graph sets eventsTruncated and mirrors at most max events', as }) } - const upsertDeliveryEvent = vi.fn(async () => ({ - inserted: true, - accepted: true, - updatedLatestStatus: false, - })) + const upsertDeliveryEvents = vi.fn( + async (input: { events: Array }) => ({ + results: input.events.map((event) => ({ + eventId: event.id, + inserted: true, + accepted: true, + })), + }), + ) const mirrorMessage = vi.fn(async () => ({ ok: true as const, accepted: true, })) const { env, writeDataPoint } = fakeMailboxEnv({ mirrorMessage, - upsertDeliveryEvent, + upsertDeliveryEvents, }) const summary = await mirrorMailboxMessageGraphFromD1({ @@ -335,16 +357,117 @@ test('live-mirror graph sets eventsTruncated and mirrors at most max events', as expect(summary.eventsTruncated).toBe(true) expect(summary.events).toHaveLength(mailboxLiveMirrorMaxEvents) - expect(summary.events[0]?.eventId).toBe('evt-000') + // Newest max kept: drop oldest overflow evt-000. + expect(summary.events[0]?.eventId).toBe('evt-001') expect(summary.events.at(-1)?.eventId).toBe( - `evt-${String(mailboxLiveMirrorMaxEvents - 1).padStart(3, '0')}`, + `evt-${String(mailboxLiveMirrorMaxEvents).padStart(3, '0')}`, + ) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + const batchEvents = upsertDeliveryEvents.mock.calls[0]?.[0]?.events ?? [] + expect(batchEvents).toHaveLength(mailboxLiveMirrorMaxEvents) + expect(batchEvents[0]?.id).toBe('evt-001') + expect(batchEvents.at(-1)?.id).toBe( + `evt-${String(mailboxLiveMirrorMaxEvents).padStart(3, '0')}`, ) - expect(upsertDeliveryEvent).toHaveBeenCalledTimes(mailboxLiveMirrorMaxEvents) - // 1 message + max events AE writes; never the overflow row. + // 1 message + 1 batch AE write; never per-event fan-out. expect(writeDataPoint).toHaveBeenCalledTimes( mailboxLiveMirrorMaxAnalyticsWrites, ) expect(writeDataPoint.mock.calls.length).toBeLessThan(250) + expect( + writeDataPoint.mock.calls.some( + (call) => + (call[0] as { blobs: Array }).blobs[0] === + 'mailbox_mirror:upsert_delivery_event_batch', + ), + ).toBe(true) +}) + +test('live-mirror graph maps batch timeout and error onto each event summary', async () => { + vi.useFakeTimers() + consoleWarn.mockImplementation(() => {}) + try { + const { db } = await createEmailDb() + await seedMessageGraph(db) + + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const hangBatch = vi.fn(() => new Promise(() => {})) + const { env: hangEnv, writeDataPoint: hangWrite } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents: hangBatch, + }) + + const pendingTimeout = mirrorMailboxMessageGraphFromD1({ + env: hangEnv, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + const timeoutExpectation = expect(pendingTimeout).resolves.toEqual({ + messageId: 'msg-1', + message: { status: 'mirrored' }, + events: [ + { eventId: 'evt-1', result: { status: 'timeout' } }, + { eventId: 'evt-2', result: { status: 'timeout' } }, + ], + eventsTruncated: false, + }) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + await timeoutExpectation + expect(hangWrite).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'timeout', + expect.any(String), + ], + }), + ) + + const failingBatch = vi.fn(async () => { + throw new Error('batch exploded') + }) + const { env: failEnv, writeDataPoint: failWrite } = fakeMailboxEnv({ + mirrorMessage, + upsertDeliveryEvents: failingBatch, + }) + const failSummary = await mirrorMailboxMessageGraphFromD1({ + env: failEnv, + db, + userId: 'user-aaa', + messageId: 'msg-1', + }) + expect(failSummary.events).toEqual([ + { + eventId: 'evt-1', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch exploded' }), + }, + }, + { + eventId: 'evt-2', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch exploded' }), + }, + }, + ]) + expect(failWrite).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'error', + expect.any(String), + ], + }), + ) + } finally { + vi.useRealTimers() + } }) test('live-mirror graph returns missing when D1 message is absent', async () => { diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts index 062f915b8b..284f20d7fc 100644 --- a/packages/worker/src/email/mailbox-live-mirror.ts +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -1,5 +1,6 @@ import { mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxDeliveryEventSnapshots, mirrorMailboxMessageSnapshot, type MailboxMirrorEnv, type MailboxMirrorResult, @@ -9,7 +10,7 @@ import { listMailboxDeliveryEventMirrorInputsForMessage, } from './mailbox-snapshot-repo.ts' import { toMailboxDeliveryEventInput } from './mailbox-snapshots.ts' -import { mailboxDefaultPageSize } from './mailbox-types.ts' +import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' import { getEmailMessageById, getEmailThreadById, @@ -27,20 +28,19 @@ import { type EmailThreadRecord } from './types.ts' * Callers that just created a thread may pass it via `thread` to avoid a * round-trip; otherwise the graph helper loads it with `getEmailThreadById`. * - * Graph event fan-out runs concurrently after the message RPC settles so event - * wall time stays near one RPC timeout (not limit × timeout). Analytics Engine - * parity writes stay under the ~250/request cap: 1 message + max events. + * Graph event mirroring uses one batch RPC after the message settles (not + * Promise.all of per-event RPCs). Analytics Engine parity writes stay small: + * 1 message + 1 batch outcome. */ /** * Max delivery events mirrored per graph call. - * `1 + mailboxLiveMirrorMaxEvents` AE writes must stay under ~250/request. + * Matches the DO batch RPC bound ({@link mailboxUpsertDeliveryEventsMax}). */ -export const mailboxLiveMirrorMaxEvents = mailboxDefaultPageSize +export const mailboxLiveMirrorMaxEvents = mailboxUpsertDeliveryEventsMax -/** AE write budget headroom check (message + event outcomes). */ -export const mailboxLiveMirrorMaxAnalyticsWrites = - 1 + mailboxLiveMirrorMaxEvents +/** AE write budget: message outcome + one batch outcome. */ +export const mailboxLiveMirrorMaxAnalyticsWrites = 2 export type MailboxLiveMirrorEnv = MailboxMirrorEnv @@ -114,16 +114,14 @@ export async function mirrorMailboxDeliveryEventFromD1(input: { /** * Load the authoritative D1 message graph and best-effort mirror it: - * message (+ optional caller-supplied thread, attachments) first, then all - * delivery events concurrently (cohesive projection; `created_at` as - * `sourceMutationAt`). + * message (+ optional caller-supplied thread, attachments) first, then a + * bounded batch of delivery events (cohesive projection; `created_at` as + * `sourceMutationAt`) via one RPC after the message settles. * - * Never throws. Returns a bounded structured summary. Queries `max+1` events - * so truncation is explicit; mirrors at most {@link mailboxLiveMirrorMaxEvents}. + * Never throws. Returns a bounded structured summary. Queries newest + * `max+1` events so truncation is explicit; mirrors at most + * {@link mailboxLiveMirrorMaxEvents} in chronological order. * - * @param sourceMutationAt - Optional caller mutation timestamp for context; - * D1 row timestamps remain authoritative. Delivery events always use - * `created_at` for immutable outbound/provider inserts. * @param thread - Optional thread from the caller. When omitted, loads via * `getEmailThreadById` when `message.threadId` is set. Explicit `null` * skips the thread snapshot. @@ -133,12 +131,8 @@ export async function mirrorMailboxMessageGraphFromD1(input: { db: D1Database userId: string messageId: string - sourceMutationAt?: string thread?: EmailThreadRecord | null }): Promise { - // Caller mutation timestamp is accepted for API symmetry with dual-write - // sites; D1 row timestamps remain authoritative for the loaded graph. - void input.sourceMutationAt try { const message = await getEmailMessageById({ db: input.db, @@ -163,7 +157,7 @@ export async function mirrorMailboxMessageGraphFromD1(input: { }) } - // Message settles before any event RPC starts (ordering guarantee). + // Message settles before the event-batch RPC starts (ordering guarantee). const messageResult = await mirrorMailboxMessageSnapshot({ env: input.env, thread, @@ -171,6 +165,8 @@ export async function mirrorMailboxMessageGraphFromD1(input: { attachments, }) + // Newest max+1 from D1 (chrono-restored). When truncated, keep the + // trailing newest max — drop the oldest overflow row at index 0. const loadedEvents = await listMailboxDeliveryEventMirrorInputsForMessage({ db: input.db, ownerId: input.userId, @@ -179,20 +175,17 @@ export async function mirrorMailboxMessageGraphFromD1(input: { }) const eventsTruncated = loadedEvents.length > mailboxLiveMirrorMaxEvents const eventInputs = eventsTruncated - ? loadedEvents.slice(0, mailboxLiveMirrorMaxEvents) + ? loadedEvents.slice(-mailboxLiveMirrorMaxEvents) : loadedEvents - // Concurrent fan-out: event-phase wall time ≈ one RPC timeout, not N×. - const events = await Promise.all( - eventInputs.map(async (event) => ({ - eventId: event.id, - result: await mirrorMailboxDeliveryEventSnapshot({ - env: input.env, - ownerId: input.userId, - event, - }), - })), - ) + const events = + eventInputs.length === 0 + ? [] + : await mirrorMailboxDeliveryEventSnapshots({ + env: input.env, + ownerId: input.userId, + events: eventInputs, + }) return { messageId: input.messageId, diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 0ede393f94..29a6bae1e6 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -4,6 +4,7 @@ import { systemEmailOwnerId } from './email-owner.ts' import { mailboxMirrorRpcTimeoutMs, mirrorMailboxDeliveryEventSnapshot, + mirrorMailboxDeliveryEventSnapshots, mirrorMailboxDeleteDeliveryEvent, mirrorMailboxDeleteMessageMetadata, mirrorMailboxDeleteThreadIfEmpty, @@ -448,3 +449,159 @@ test('mailbox mirror RPC timeout returns timeout and late rejection is not unhan vi.useRealTimers() } }) + +function baseDeliveryEventInput( + overrides?: Partial, +): MailboxDeliveryEventInput { + return { + id: 'evt-1', + messageId: 'msg-1', + inboxId: 'inbox-1', + eventType: 'sent', + provider: 'kody', + providerMessageId: null, + providerEventId: null, + detailJson: '{}', + needsEffectReconcile: false, + state: null, + fingerprint: null, + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: null, + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: null, + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: '2026-07-02T10:00:00.000Z', + updatedAt: '2026-07-02T10:00:00.000Z', + ...overrides, + } +} + +test('mailbox mirror batch helper uses one RPC, one telemetry outcome, and maps per-event results', async () => { + consoleWarn.mockImplementation(() => {}) + const events = [ + baseDeliveryEventInput({ id: 'evt-a' }), + baseDeliveryEventInput({ + id: 'evt-b', + createdAt: '2026-07-02T10:00:01.000Z', + updatedAt: '2026-07-02T10:00:01.000Z', + }), + ] + const upsertDeliveryEvents = vi.fn(async () => ({ + results: [ + { eventId: 'evt-a', inserted: true, accepted: true }, + { eventId: 'evt-b', inserted: false, accepted: false }, + ], + })) + const { env, writeDataPoint } = fakeMailboxEnv({ upsertDeliveryEvents }) + + await expect( + mirrorMailboxDeliveryEventSnapshots({ + env, + ownerId: 'user-aaa', + events, + }), + ).resolves.toEqual([ + { eventId: 'evt-a', result: { status: 'mirrored' } }, + { eventId: 'evt-b', result: { status: 'stale' } }, + ]) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) + expect(upsertDeliveryEvents).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + events, + }) + expect(writeDataPoint).toHaveBeenCalledTimes(1) + expect(writeDataPoint).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'stale', + expect.any(String), + ], + }), + ) + + await expect( + mirrorMailboxDeliveryEventSnapshots({ + env, + ownerId: 'user-aaa', + events: [], + }), + ).resolves.toEqual([]) + expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) +}) + +test('mailbox mirror batch helper maps timeout and error to every event', async () => { + vi.useFakeTimers() + consoleWarn.mockImplementation(() => {}) + try { + const events = [ + baseDeliveryEventInput({ id: 'evt-1' }), + baseDeliveryEventInput({ id: 'evt-2' }), + ] + const hang = vi.fn(() => new Promise(() => {})) + const { env: hangEnv, writeDataPoint: hangWrite } = fakeMailboxEnv({ + upsertDeliveryEvents: hang, + }) + const pending = mirrorMailboxDeliveryEventSnapshots({ + env: hangEnv, + ownerId: 'user-aaa', + events, + }) + const expectation = expect(pending).resolves.toEqual([ + { eventId: 'evt-1', result: { status: 'timeout' } }, + { eventId: 'evt-2', result: { status: 'timeout' } }, + ]) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + await expectation + expect(hangWrite).toHaveBeenCalledTimes(1) + + const fail = vi.fn(async () => { + throw new Error('batch unavailable') + }) + const { env: failEnv } = fakeMailboxEnv({ upsertDeliveryEvents: fail }) + await expect( + mirrorMailboxDeliveryEventSnapshots({ + env: failEnv, + ownerId: 'user-aaa', + events, + }), + ).resolves.toEqual([ + { + eventId: 'evt-1', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch unavailable' }), + }, + }, + { + eventId: 'evt-2', + result: { + status: 'error', + error: expect.objectContaining({ message: 'batch unavailable' }), + }, + }, + ]) + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index 4a1c2b6bd3..b3869dd206 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -31,6 +31,7 @@ import { type MailboxSetMessageClassificationInput, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, + type MailboxUpsertDeliveryEventsResult, } from './mailbox-types.ts' /** @@ -68,6 +69,11 @@ export type MailboxMirrorResult = | { status: 'skipped'; reason: MailboxMirrorSkipReason } | { status: 'error'; error: unknown } +export type MailboxMirrorDeliveryEventBatchItemResult = { + eventId: string + result: MailboxMirrorResult +} + function resolveMirrorOwner( ownerId: string | null | undefined, ): { ok: true; ownerId: string } | { ok: false; result: MailboxMirrorResult } { @@ -169,6 +175,14 @@ function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { return accepted ? { status: 'mirrored' } : { status: 'stale' } } +function outcomeFromBatchAccepted( + results: MailboxUpsertDeliveryEventsResult['results'], +): MailboxMirrorResult { + return results.every((item) => item.accepted) + ? { status: 'mirrored' } + : { status: 'stale' } +} + /** * Race a DO RPC against {@link mailboxMirrorRpcTimeoutMs}. * The settled wrapper ensures a late rejection cannot become unhandled. @@ -203,32 +217,51 @@ export async function awaitMailboxMirrorRpc( } } -async function runMirror(input: { +/** + * Shared owner/config/timeout/telemetry pipeline for single-result mirror RPCs. + * Thin wrappers supply only the RPC call and result mapping. + */ +async function executeMailboxMirrorRpc(input: { tag: string env: MailboxMirrorEnv - userId: string | null | undefined + ownerId: string | null | undefined operation: MailboxMirrorOperation - run: () => Promise + call: (ownerId: string) => Promise + toResult: (value: T) => MailboxMirrorResult }): Promise { + const owner = resolveMirrorOwner(input.ownerId) + if (!owner.ok) { + recordMirrorOutcome(input.env, input.ownerId, input.operation, owner.result) + return owner.result + } + const skippedMailbox = skipIfMailboxMissing(input.env) + if (skippedMailbox) { + recordMirrorOutcome( + input.env, + owner.ownerId, + input.operation, + skippedMailbox, + ) + return skippedMailbox + } + let result: MailboxMirrorResult try { - result = await input.run() + const raced = await awaitMailboxMirrorRpc(input.call(owner.ownerId)) + result = raced.ok ? input.toResult(raced.value) : { status: 'timeout' } } catch (error) { console.warn(input.tag, error) result = { status: 'error', error } } - recordMirrorOutcome(input.env, input.userId, input.operation, result) + recordMirrorOutcome(input.env, owner.ownerId, input.operation, result) return result } -function finishEarly(input: { - env: MailboxMirrorEnv - userId: string | null | undefined - operation: MailboxMirrorOperation - result: MailboxMirrorResult -}): MailboxMirrorResult { - recordMirrorOutcome(input.env, input.userId, input.operation, input.result) - return input.result +function mapUniformBatchResults( + events: Array<{ id: string }>, + result: MailboxMirrorResult, +): Array { + return events.map((event) => ({ eventId: event.id, result })) } /** Best-effort full message (+ optional thread/attachments) snapshot mirror. */ @@ -238,46 +271,19 @@ export async function mirrorMailboxMessageSnapshot(input: { message: EmailMessageRecord attachments?: Array }): Promise { - const operation = 'mirror_message' as const - const owner = resolveMirrorOwner(input.message.userId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.message.userId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-message-failed', env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).mirrorMessage({ - ownerId: owner.ownerId, - thread: input.thread ? toMailboxThreadInput(input.thread) : null, - message: toMailboxMessageInput(input.message), - attachments: input.attachments?.map(toMailboxAttachmentInput), - }), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromAccepted(raced.value.accepted) - }, + ownerId: input.message.userId, + operation: 'mirror_message', + call: (ownerId) => + mailboxRpc({ env: input.env, userId: ownerId }).mirrorMessage({ + ownerId, + thread: input.thread ? toMailboxThreadInput(input.thread) : null, + message: toMailboxMessageInput(input.message), + attachments: input.attachments?.map(toMailboxAttachmentInput), + }), + toResult: (value) => outcomeFromAccepted(value.accepted), }) } @@ -298,88 +304,101 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { deliveryStatusAt: string } | null }): Promise { - const operation = 'upsert_delivery_event' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delivery-event-failed', env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).upsertDeliveryEvent({ - ownerId: owner.ownerId, - event: input.event, - latestDeliveryStatus: input.latestDeliveryStatus, - }), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromAccepted(raced.value.accepted) - }, + ownerId: input.ownerId, + operation: 'upsert_delivery_event', + call: (ownerId) => + mailboxRpc({ env: input.env, userId: ownerId }).upsertDeliveryEvent({ + ownerId, + event: input.event, + latestDeliveryStatus: input.latestDeliveryStatus, + }), + toResult: (value) => outcomeFromAccepted(value.accepted), }) } -/** Best-effort thread touch mirror. */ -export async function mirrorMailboxTouchThread( - input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, -): Promise { - const operation = 'touch_thread' as const +/** + * Best-effort batch delivery-event snapshot mirror. + * + * One 1s timeout and one `upsert_delivery_event_batch` telemetry outcome. + * Timeout/error/skip apply uniformly to each per-event summary entry. + * Empty `events` returns `[]` without RPC or telemetry. + */ +export async function mirrorMailboxDeliveryEventSnapshots(input: { + env: MailboxMirrorEnv + ownerId: string + events: Array +}): Promise> { + const operation = 'upsert_delivery_event_batch' as const + if (input.events.length === 0) return [] + const owner = resolveMirrorOwner(input.ownerId) if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) + recordMirrorOutcome(input.env, input.ownerId, operation, owner.result) + return mapUniformBatchResults(input.events, owner.result) } const skippedMailbox = skipIfMailboxMissing(input.env) if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, + recordMirrorOutcome(input.env, owner.ownerId, operation, skippedMailbox) + return mapUniformBatchResults(input.events, skippedMailbox) + } + + try { + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }).upsertDeliveryEvents({ + ownerId: owner.ownerId, + events: input.events, + }), + ) + if (!raced.ok) { + const result = { status: 'timeout' as const } + recordMirrorOutcome(input.env, owner.ownerId, operation, result) + return mapUniformBatchResults(input.events, result) + } + recordMirrorOutcome( + input.env, + owner.ownerId, operation, - result: skippedMailbox, + outcomeFromBatchAccepted(raced.value.results), + ) + const byEventId = new Map( + raced.value.results.map((item) => [item.eventId, item] as const), + ) + return input.events.map((event) => { + const item = byEventId.get(event.id) + return { + eventId: event.id, + result: item + ? outcomeFromAccepted(item.accepted) + : { status: 'missing' as const }, + } }) + } catch (error) { + console.warn('mailbox-mirror-delivery-event-batch-failed', error) + const result = { status: 'error' as const, error } + recordMirrorOutcome(input.env, owner.ownerId, operation, result) + return mapUniformBatchResults(input.events, result) } +} - return runMirror({ +/** Best-effort thread touch mirror. */ +export async function mirrorMailboxTouchThread( + input: MailboxTouchThreadInput & { env: MailboxMirrorEnv }, +): Promise { + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-touch-thread-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).touchThread(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromPartialMutation(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'touch_thread', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).touchThread(rpcInput), + toResult: outcomeFromPartialMutation, }) } @@ -387,42 +406,15 @@ export async function mirrorMailboxTouchThread( export async function mirrorMailboxUpdateMessageDelivery( input: MailboxUpdateMessageDeliveryInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'update_message_delivery' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-update-message-delivery-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).updateMessageDelivery(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromPartialMutation(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'update_message_delivery', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).updateMessageDelivery(rpcInput), + toResult: outcomeFromPartialMutation, }) } @@ -430,42 +422,15 @@ export async function mirrorMailboxUpdateMessageDelivery( export async function mirrorMailboxSetMessageClassification( input: MailboxSetMessageClassificationInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'set_message_classification' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-set-message-classification-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).setMessageClassification(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromPartialMutation(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'set_message_classification', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).setMessageClassification(rpcInput), + toResult: outcomeFromPartialMutation, }) } @@ -473,42 +438,15 @@ export async function mirrorMailboxSetMessageClassification( export async function mirrorMailboxDeleteMessageMetadata( input: MailboxDeleteMessageMetadataInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'delete_message_metadata' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delete-message-metadata-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteMessageMetadata(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromDeleteResult(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'delete_message_metadata', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).deleteMessageMetadata(rpcInput), + toResult: outcomeFromDeleteResult, }) } @@ -516,42 +454,15 @@ export async function mirrorMailboxDeleteMessageMetadata( export async function mirrorMailboxDeleteDeliveryEvent( input: MailboxDeleteDeliveryEventInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'delete_delivery_event' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delete-delivery-event-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteDeliveryEvent(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromDeleteResult(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'delete_delivery_event', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).deleteDeliveryEvent(rpcInput), + toResult: outcomeFromDeleteResult, }) } @@ -559,41 +470,14 @@ export async function mirrorMailboxDeleteDeliveryEvent( export async function mirrorMailboxDeleteThreadIfEmpty( input: MailboxDeleteThreadIfEmptyInput & { env: MailboxMirrorEnv }, ): Promise { - const operation = 'delete_thread_if_empty' as const - const owner = resolveMirrorOwner(input.ownerId) - if (!owner.ok) { - return finishEarly({ - env: input.env, - userId: input.ownerId, - operation, - result: owner.result, - }) - } - const skippedMailbox = skipIfMailboxMissing(input.env) - if (skippedMailbox) { - return finishEarly({ - env: input.env, - userId: owner.ownerId, - operation, - result: skippedMailbox, - }) - } - - return runMirror({ + const { env, ...rpcInput } = input + return executeMailboxMirrorRpc({ tag: 'mailbox-mirror-delete-thread-if-empty-failed', - env: input.env, - userId: owner.ownerId, - operation, - run: async () => { - const { env: _env, ...rpcInput } = input - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).deleteThreadIfEmpty(rpcInput), - ) - if (!raced.ok) return { status: 'timeout' } - return outcomeFromDeleteResult(raced.value) - }, + env, + ownerId: input.ownerId, + operation: 'delete_thread_if_empty', + call: (ownerId) => + mailboxRpc({ env, userId: ownerId }).deleteThreadIfEmpty(rpcInput), + toResult: outcomeFromDeleteResult, }) } diff --git a/packages/worker/src/email/mailbox-parity-events.node.test.ts b/packages/worker/src/email/mailbox-parity-events.node.test.ts index 70cebc6313..0db8924499 100644 --- a/packages/worker/src/email/mailbox-parity-events.node.test.ts +++ b/packages/worker/src/email/mailbox-parity-events.node.test.ts @@ -53,6 +53,13 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = outcome: 'error', timestamp: '2026-07-30T10:09:00.000Z', }) + recordMailboxParityEvent(env, { + userId: 'user-1', + category: 'mirror', + operation: 'upsert_delivery_event_batch', + outcome: 'timeout', + timestamp: '2026-07-30T10:10:00.000Z', + }) recordMailboxParityEvent(env, { userId: 'user-2', category: 'parity', @@ -70,7 +77,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = timestamp: '2026-07-30T12:00:00.000Z', }) - expect(writeDataPoint).toHaveBeenCalledTimes(8) + expect(writeDataPoint).toHaveBeenCalledTimes(9) expect(writeDataPoint).toHaveBeenNthCalledWith(1, { indexes: ['user-1'], blobs: [ @@ -122,6 +129,15 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = doubles: [1, 0], }) expect(writeDataPoint).toHaveBeenNthCalledWith(7, { + indexes: ['user-1'], + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'timeout', + '2026-07-30T10:10:00.000Z', + ], + doubles: [1, 0], + }) + expect(writeDataPoint).toHaveBeenNthCalledWith(8, { indexes: ['user-2'], blobs: [ 'mailbox_parity:compare_messages', @@ -130,7 +146,7 @@ test('mailbox parity writes compact indexed mirror and parity data points', () = ], doubles: [1, 2], }) - expect(writeDataPoint).toHaveBeenNthCalledWith(8, { + expect(writeDataPoint).toHaveBeenNthCalledWith(9, { indexes: ['user-3'], blobs: [ 'mailbox_parity:compare_threads', diff --git a/packages/worker/src/email/mailbox-parity-events.ts b/packages/worker/src/email/mailbox-parity-events.ts index bb2c564770..fde129a0ac 100644 --- a/packages/worker/src/email/mailbox-parity-events.ts +++ b/packages/worker/src/email/mailbox-parity-events.ts @@ -7,6 +7,7 @@ export type MailboxParityEnv = { export type MailboxMirrorOperation = | 'mirror_message' | 'upsert_delivery_event' + | 'upsert_delivery_event_batch' | 'touch_thread' | 'update_message_delivery' | 'set_message_classification' diff --git a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts index 2357861335..d79608b39f 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts @@ -172,4 +172,13 @@ test('mailbox-snapshot-repo loads complete delivery-event projection and ready M updatedAt: '2026-07-02T10:05:00.000Z', }), ]) + + // Newest-N selection: limit 1 returns the newest event, chrono-restored. + const newestOnly = await listMailboxDeliveryEventMirrorInputsForMessage({ + db, + ownerId: 'user-aaa', + messageId: 'msg-1', + limit: 1, + }) + expect(newestOnly.map((event) => event.id)).toEqual(['evt-2']) }) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index 9d732c35a3..03d69a8fd8 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -128,8 +128,11 @@ export async function getMailboxDeliveryEventMirrorInput(input: { } /** - * Load complete delivery-event mirror projections for one message, oldest - * first, bounded by `limit`. Scoped by owner. + * Load complete delivery-event mirror projections for one message. + * + * Selects the newest `limit` rows (`ORDER BY created_at DESC, id DESC`) so + * truncation keeps recent events, then restores chronological order for + * insertion. Scoped by owner. */ export async function listEmailDeliveryEventMirrorProjectionsForMessage(input: { db: D1Database @@ -142,12 +145,13 @@ export async function listEmailDeliveryEventMirrorProjectionsForMessage(input: { `${deliveryEventMirrorSelect} WHERE message_id = ? AND user_id = ? - ORDER BY created_at ASC, id ASC + ORDER BY created_at DESC, id DESC LIMIT ?`, ) .bind(input.messageId, input.ownerId, input.limit) .all() - return (result.results ?? []).map(mapDeliveryEventMirrorRow) + const newestFirst = (result.results ?? []).map(mapDeliveryEventMirrorRow) + return newestFirst.reverse() } /** diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 5855ebce86..f506665e2d 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -20,6 +20,8 @@ export const mailboxMessageRetentionDays = 365 export const mailboxDeliveryEventRetentionDays = 90 export const mailboxDefaultPageSize = 100 export const mailboxMaxPageSize = 500 +/** Max complete delivery-event snapshots per `upsertDeliveryEvents` RPC. */ +export const mailboxUpsertDeliveryEventsMax = mailboxDefaultPageSize export const mailboxRetentionBatchSize = 100 export const mailboxRetentionAlarmSkewMs = 60_000 /** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ @@ -434,6 +436,17 @@ export type MailboxDeleteResult = | { status: 'missing' } | { status: 'stale' } +/** Per-event outcome from {@link MailboxRpc.upsertDeliveryEvents}. */ +export type MailboxUpsertDeliveryEventBatchItemResult = { + eventId: string + inserted: boolean + accepted: boolean +} + +export type MailboxUpsertDeliveryEventsResult = { + results: Array +} + export type MailboxRpc = { mirrorMessage: (input: { /** @@ -463,6 +476,15 @@ export type MailboxRpc = { accepted: boolean updatedLatestStatus: boolean }> + /** + * Upsert a bounded batch of complete immutable delivery-event snapshots in + * one transaction. Does not patch message latest delivery status — that + * remains from the prior full message snapshot. + */ + upsertDeliveryEvents: (input: { + ownerId: string + events: Array + }) => Promise touchThread: ( input: MailboxTouchThreadInput, ) => Promise diff --git a/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts new file mode 100644 index 0000000000..c93610214b --- /dev/null +++ b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts @@ -0,0 +1,277 @@ +import { env } from 'cloudflare:workers' +import { expect, test } from 'vitest' +import { http, HttpResponse } from 'msw' +import { bytesToBase64 } from '@kody-internal/shared/base64.ts' +import { ensureEmailTestSchema } from './test-schema.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import { rpcFor } from './mailbox-test-helpers.ts' +import { getEmailMessageById } from './repo.ts' +import { sendOutboundEmail } from './outbound.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createMswNodeServer } from '#worker/test-support/msw-node-server.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' + +const cloudflareEmailApi = + 'https://api.cloudflare.test/client/v4/accounts/account-123/email/sending/send' + +const platformBaseUrl = 'https://kody.example.com' +// User mail lives on the inbox. subdomain derived from APP_BASE_URL. +const platformDomain = 'inbox.kody.example.com' + +/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ +function createMailboxStubEnv(input: { + mode: 'throw' | 'hang' + base?: typeof env +}) { + const base = input.base ?? env + const hang = () => new Promise(() => {}) + const fail = async () => { + throw new Error('simulated mailbox failure') + } + const method = input.mode === 'hang' ? hang : fail + const stub = { + mirrorMessage: method, + upsertDeliveryEvent: method, + touchThread: method, + updateMessageDelivery: method, + setMessageClassification: method, + deleteMessageMetadata: method, + deleteDeliveryEvent: method, + deleteThreadIfEmpty: method, + } + return { + ...base, + APP_BASE_URL: platformBaseUrl, + MAILBOX: { + idFromName: (name: string) => base.MAILBOX.idFromName(name), + get: () => stub, + } as unknown as DurableObjectNamespace, + } +} + +async function seedVerifiedAccount(input: { email: string }) { + const username = `sender-${crypto.randomUUID().slice(0, 8)}` + const stableUserId = await createStableUserIdFromEmail(input.email) + await env.APP_DB.prepare( + `INSERT INTO users (username, email, password_hash, email_verified_at, plan, stable_user_id) + VALUES (?, ?, ?, ?, 'max', ?)`, + ) + .bind( + username, + input.email, + 'test-password-hash', + new Date().toISOString(), + stableUserId, + ) + .run() + return { username, from: `${username}@${platformDomain}` } +} + +test('sendOutboundEmail mirrors outbound message graph into Mailbox on success', async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-sent-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]) + const sendEnv = { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-sent' } + }, + }, + } + + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror sent', + text: 'Mirrored body', + attachments: [ + { + filename: 'note.pdf', + contentType: 'application/pdf', + contentBase64: bytesToBase64(pdfBytes), + }, + ], + }) + + expect(result.status).toBe('sent') + expect(result.message.threadId).toBeTruthy() + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'sent', + providerMessageId: 'provider-mailbox-sent', + subject: 'Mailbox mirror sent', + threadId: result.message.threadId, + }) + const mirroredThread = await mailbox.getThread({ + threadId: result.message.threadId!, + }) + expect(mirroredThread).toMatchObject({ + id: result.message.threadId, + subjectNormalized: 'mailbox mirror sent', + }) + const mirroredAttachments = await mailbox.listAttachmentsForMessage({ + messageId: result.message.id, + }) + expect(mirroredAttachments).toEqual([ + expect.objectContaining({ + filename: 'note.pdf', + contentType: 'application/pdf', + size: pdfBytes.byteLength, + storageKind: 'external', + }), + ]) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'send_requested', + 'sent', + ]) +}, 30_000) + +test('sendOutboundEmail mirrors failed outbound graph into Mailbox', async () => { + silenceIncidentalRuntimeWarnings(['cloudflare-email-api-failed']) + await ensureEmailTestSchema(env.APP_DB) + const accountEmail = `mailbox-fail-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + + using _server = createMswNodeServer( + [ + http.post(cloudflareEmailApi, () => + HttpResponse.json( + { success: false, errors: [{ message: 'provider down' }] }, + { status: 500 }, + ), + ), + ], + { onUnhandledRequest: 'bypass' }, + ) + + const result = await sendOutboundEmail({ + env: { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: undefined as unknown as SendEmail, + CLOUDFLARE_ACCOUNT_ID: 'account-123', + CLOUDFLARE_API_BASE_URL: 'https://api.cloudflare.test', + CLOUDFLARE_API_TOKEN: 'token-123', + }, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox mirror failed', + text: 'Body', + }) + + expect(result.status).toBe('failed') + expect(result.error).toBe('provider down') + const mailbox = rpcFor(userId) + const mirroredMessage = await mailbox.getMessage({ + messageId: result.message.id, + }) + expect(mirroredMessage).toMatchObject({ + id: result.message.id, + direction: 'outbound', + processingStatus: 'failed', + error: 'provider down', + threadId: result.message.threadId, + }) + expect( + await mailbox.getThread({ threadId: result.message.threadId! }), + ).toMatchObject({ id: result.message.threadId }) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: result.message.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ + 'failed', + 'send_requested', + ]) +}, 30_000) + +test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-mirror-message-failed', + 'mailbox-mirror-delivery-event-failed', + 'mailbox-live-mirror-message-graph-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + + { + const accountEmail = `mailbox-err-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'throw' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-err' } + }, + }, + } + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox throw ignored', + text: 'Body', + }) + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-err', + error: null, + }) + const stored = await getEmailMessageById({ + db: env.APP_DB, + userId, + messageId: result.message.id, + }) + expect(stored?.processingStatus).toBe('sent') + } + + { + const accountEmail = `mailbox-hang-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + await seedVerifiedAccount({ email: accountEmail }) + const sendEnv = { + ...createMailboxStubEnv({ mode: 'hang' }), + EMAIL: { + async send() { + return { messageId: 'provider-mailbox-hang' } + }, + }, + } + const startedAt = Date.now() + const result = await sendOutboundEmail({ + env: sendEnv, + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Mailbox hang ignored', + text: 'Body', + }) + const elapsedMs = Date.now() - startedAt + expect(result).toMatchObject({ + status: 'sent', + providerMessageId: 'provider-mailbox-hang', + error: null, + }) + // Message + send_requested + sent each race the mirror RPC bound. + expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 6) + } +}, 30_000) diff --git a/packages/worker/src/email/outbound.workers.test.ts b/packages/worker/src/email/outbound.workers.test.ts index a1be15b7f7..3f87a19883 100644 --- a/packages/worker/src/email/outbound.workers.test.ts +++ b/packages/worker/src/email/outbound.workers.test.ts @@ -6,8 +6,6 @@ import { bytesToBase64 } from '@kody-internal/shared/base64.ts' import { McpCallerError } from '#mcp/caller-error.ts' import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' import { ensureEmailTestSchema } from './test-schema.ts' -import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' -import { rpcFor } from './mailbox-test-helpers.ts' import { deleteEmailMessageById, getEmailMessageById, @@ -50,37 +48,6 @@ function createBindingSendEnv() { } } -/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ -function createMailboxStubEnv(input: { - mode: 'throw' | 'hang' - base?: typeof env -}) { - const base = input.base ?? env - const hang = () => new Promise(() => {}) - const fail = async () => { - throw new Error('simulated mailbox failure') - } - const method = input.mode === 'hang' ? hang : fail - const stub = { - mirrorMessage: method, - upsertDeliveryEvent: method, - touchThread: method, - updateMessageDelivery: method, - setMessageClassification: method, - deleteMessageMetadata: method, - deleteDeliveryEvent: method, - deleteThreadIfEmpty: method, - } - return { - ...base, - APP_BASE_URL: platformBaseUrl, - MAILBOX: { - idFromName: (name: string) => base.MAILBOX.idFromName(name), - get: () => stub, - } as unknown as DurableObjectNamespace, - } -} - async function seedVerifiedAccount(input: { email: string username?: string @@ -999,212 +966,3 @@ test('sendOutboundEmail caps max-plan users at the email daily backstop', async maxPlanEmailLimits.email_sends_per_day, ) }, 30_000) - -test('sendOutboundEmail mirrors outbound message graph into Mailbox on success', async () => { - silenceIncidentalRuntimeWarnings() - await ensureEmailTestSchema(env.APP_DB) - const accountEmail = `mailbox-sent-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]) - const sendEnv = { - ...env, - APP_BASE_URL: platformBaseUrl, - EMAIL: { - async send() { - return { messageId: 'provider-mailbox-sent' } - }, - }, - } - - const result = await sendOutboundEmail({ - env: sendEnv, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox mirror sent', - text: 'Mirrored body', - attachments: [ - { - filename: 'note.pdf', - contentType: 'application/pdf', - contentBase64: bytesToBase64(pdfBytes), - }, - ], - }) - - expect(result.status).toBe('sent') - expect(result.message.threadId).toBeTruthy() - const mailbox = rpcFor(userId) - const mirroredMessage = await mailbox.getMessage({ - messageId: result.message.id, - }) - expect(mirroredMessage).toMatchObject({ - id: result.message.id, - direction: 'outbound', - processingStatus: 'sent', - providerMessageId: 'provider-mailbox-sent', - subject: 'Mailbox mirror sent', - threadId: result.message.threadId, - }) - const mirroredThread = await mailbox.getThread({ - threadId: result.message.threadId!, - }) - expect(mirroredThread).toMatchObject({ - id: result.message.threadId, - subjectNormalized: 'mailbox mirror sent', - }) - const mirroredAttachments = await mailbox.listAttachmentsForMessage({ - messageId: result.message.id, - }) - expect(mirroredAttachments).toEqual([ - expect.objectContaining({ - filename: 'note.pdf', - contentType: 'application/pdf', - size: pdfBytes.byteLength, - storageKind: 'external', - }), - ]) - const mirroredEvents = await mailbox.listDeliveryEvents({ - messageId: result.message.id, - limit: 10, - }) - expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ - 'send_requested', - 'sent', - ]) -}, 30_000) - -test('sendOutboundEmail mirrors failed outbound graph into Mailbox', async () => { - silenceIncidentalRuntimeWarnings(['cloudflare-email-api-failed']) - await ensureEmailTestSchema(env.APP_DB) - const accountEmail = `mailbox-fail-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - - using _server = createMswNodeServer( - [ - http.post(cloudflareEmailApi, () => - HttpResponse.json( - { success: false, errors: [{ message: 'provider down' }] }, - { status: 500 }, - ), - ), - ], - { onUnhandledRequest: 'bypass' }, - ) - - const result = await sendOutboundEmail({ - env: { - ...env, - APP_BASE_URL: platformBaseUrl, - EMAIL: undefined as unknown as SendEmail, - CLOUDFLARE_ACCOUNT_ID: 'account-123', - CLOUDFLARE_API_BASE_URL: 'https://api.cloudflare.test', - CLOUDFLARE_API_TOKEN: 'token-123', - }, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox mirror failed', - text: 'Body', - }) - - expect(result.status).toBe('failed') - expect(result.error).toBe('provider down') - const mailbox = rpcFor(userId) - const mirroredMessage = await mailbox.getMessage({ - messageId: result.message.id, - }) - expect(mirroredMessage).toMatchObject({ - id: result.message.id, - direction: 'outbound', - processingStatus: 'failed', - error: 'provider down', - threadId: result.message.threadId, - }) - expect( - await mailbox.getThread({ threadId: result.message.threadId! }), - ).toMatchObject({ id: result.message.threadId }) - const mirroredEvents = await mailbox.listDeliveryEvents({ - messageId: result.message.id, - limit: 10, - }) - expect(mirroredEvents.map((event) => event.eventType).sort()).toEqual([ - 'failed', - 'send_requested', - ]) -}, 30_000) - -test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { - silenceIncidentalRuntimeWarnings([ - 'mailbox-mirror-message-failed', - 'mailbox-mirror-delivery-event-failed', - 'mailbox-live-mirror-message-graph-failed', - ]) - await ensureEmailTestSchema(env.APP_DB) - - { - const accountEmail = `mailbox-err-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - const sendEnv = { - ...createMailboxStubEnv({ mode: 'throw' }), - EMAIL: { - async send() { - return { messageId: 'provider-mailbox-err' } - }, - }, - } - const result = await sendOutboundEmail({ - env: sendEnv, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox throw ignored', - text: 'Body', - }) - expect(result).toMatchObject({ - status: 'sent', - providerMessageId: 'provider-mailbox-err', - error: null, - }) - const stored = await getEmailMessageById({ - db: env.APP_DB, - userId, - messageId: result.message.id, - }) - expect(stored?.processingStatus).toBe('sent') - } - - { - const accountEmail = `mailbox-hang-${crypto.randomUUID()}@example.com` - const userId = await createStableUserIdFromEmail(accountEmail) - await seedVerifiedAccount({ email: accountEmail }) - const sendEnv = { - ...createMailboxStubEnv({ mode: 'hang' }), - EMAIL: { - async send() { - return { messageId: 'provider-mailbox-hang' } - }, - }, - } - const startedAt = Date.now() - const result = await sendOutboundEmail({ - env: sendEnv, - userId, - accountEmail, - recipientPolicy: 'self', - subject: 'Mailbox hang ignored', - text: 'Body', - }) - const elapsedMs = Date.now() - startedAt - expect(result).toMatchObject({ - status: 'sent', - providerMessageId: 'provider-mailbox-hang', - error: null, - }) - // Message + send_requested + sent each race the mirror RPC bound. - expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 6) - } -}, 30_000) diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index c7fc822a29..0327befafc 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -855,11 +855,13 @@ export async function getEmailMessageById(input: { } /** - * Reclassify a stored inbound message (spam <-> not spam). Reclassifying to + * D1-only inbound classification update. Prefer + * `setEmailMessageClassification` in `service.ts`, which dual-writes the + * Mailbox graph mirror after a successful mutation. Reclassifying to * `accepted` does not retroactively dispatch package subscription events; the * receive-time classification decides dispatch exactly once. */ -export async function setEmailMessageClassification(input: { +export async function updateEmailMessageClassificationInD1(input: { db: D1Database userId: string messageId: string diff --git a/packages/worker/src/email/service-classification.node.test.ts b/packages/worker/src/email/service-classification.node.test.ts new file mode 100644 index 0000000000..bc53f84752 --- /dev/null +++ b/packages/worker/src/email/service-classification.node.test.ts @@ -0,0 +1,119 @@ +import { expect, test, vi } from 'vitest' +import type * as EmailRepo from './repo.ts' + +const mocks = vi.hoisted(() => ({ + updateEmailMessageClassificationInD1: vi.fn(), + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'message-1', + message: { status: 'mirrored' as const }, + events: [], + })), +})) + +vi.mock('./repo.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + updateEmailMessageClassificationInD1: + mocks.updateEmailMessageClassificationInD1, + } +}) + +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, +})) + +const { setEmailMessageClassification } = await import('./service.ts') + +function createEnv() { + return { + APP_DB: {} as D1Database, + } as Env +} + +test('setEmailMessageClassification mutates D1 then mirrors only on success', async () => { + const env = createEnv() + const callOrder: Array = [] + mocks.updateEmailMessageClassificationInD1.mockImplementation(async () => { + callOrder.push('d1') + return true + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { + callOrder.push('mirror') + return { + messageId: 'message-1', + message: { status: 'mirrored' as const }, + events: [], + } + }) + + await expect( + setEmailMessageClassification({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + classification: 'quarantined', + classificationReason: 'Reclassified by user.', + }), + ).resolves.toBe(true) + + expect(callOrder).toEqual(['d1', 'mirror']) + expect(mocks.updateEmailMessageClassificationInD1).toHaveBeenCalledWith({ + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + classification: 'quarantined', + classificationReason: 'Reclassified by user.', + now: undefined, + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + }) +}) + +test('setEmailMessageClassification skips mirror when D1 updates nothing', async () => { + const env = createEnv() + mocks.updateEmailMessageClassificationInD1.mockResolvedValueOnce(false) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + + await expect( + setEmailMessageClassification({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'missing', + classification: 'accepted', + classificationReason: null, + }), + ).resolves.toBe(false) + + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() +}) + +test('setEmailMessageClassification still returns true when mirror reports error', async () => { + const env = createEnv() + mocks.updateEmailMessageClassificationInD1.mockResolvedValueOnce(true) + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'message-1', + message: { + status: 'error', + error: new Error('mailbox mirror timed out'), + }, + events: [], + }) + + await expect( + setEmailMessageClassification({ + env, + db: env.APP_DB, + userId: 'user-1', + messageId: 'message-1', + classification: 'quarantined', + classificationReason: 'Reclassified by user.', + }), + ).resolves.toBe(true) +}) diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index d10de66ed1..2dc1d41770 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -9,6 +9,10 @@ import { markInboundDeliveryReceived, type InboundDelivery, } from './inbound-delivery.ts' +import { + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, +} from './mailbox-live-mirror.ts' import { recordEmailReportingEvent, type EmailReportingEnv, @@ -26,6 +30,7 @@ import { insertEmailMessage, listEmailAttachmentsForMessage, touchEmailThread, + updateEmailMessageClassificationInD1, } from './repo.ts' import { evaluateEmailSenderRules } from './sender-rules.ts' import { @@ -702,6 +707,40 @@ export async function recordBoundedEmailRejectionEvent(input: { return rejectionsToday } +/** + * Reclassify a stored inbound message and, on D1 success only, best-effort + * mirror the full message graph into Mailbox. Returns the D1 mutation boolean; + * mirror failures never throw or change that response. + */ +export async function setEmailMessageClassification(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + messageId: string + classification: EmailClassification + classificationReason?: string | null + now?: string +}) { + const updated = await updateEmailMessageClassificationInD1({ + db: input.db, + userId: input.userId, + messageId: input.messageId, + classification: input.classification, + classificationReason: input.classificationReason, + now: input.now, + }) + if (!updated) { + return false + } + await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.db, + userId: input.userId, + messageId: input.messageId, + }) + return true +} + export async function recordProviderEmailDeliveryEvent(input: { db: D1Database reportingEnv?: EmailReportingEnv diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts index 35f2299626..6df5845f83 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.node.test.ts @@ -3,21 +3,12 @@ import { createMcpCallerContext } from '#mcp/context.ts' const mocks = vi.hoisted(() => ({ setEmailMessageClassification: vi.fn(), - mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ - messageId: 'message-1', - message: { status: 'mirrored' }, - events: [], - })), })) -vi.mock('#worker/email/repo.ts', () => ({ +vi.mock('#worker/email/service.ts', () => ({ setEmailMessageClassification: mocks.setEmailMessageClassification, })) -vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, -})) - const { emailMessageClassifyCapability } = await import('./email-message-classify.ts') @@ -59,21 +50,15 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'quarantined', }) expect(mocks.setEmailMessageClassification).toHaveBeenCalledWith({ + env: context.env, db: context.env.APP_DB, userId: 'user-1', messageId: 'message-1', classification: 'quarantined', classificationReason: 'Reclassified by user.', }) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env: context.env, - db: context.env.APP_DB, - userId: 'user-1', - messageId: 'message-1', - }) mocks.setEmailMessageClassification.mockResolvedValueOnce(true) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'message-1', classification: 'accepted' }, @@ -84,45 +69,19 @@ test('email_message_classify updates classification and reports not-found', asyn classification: 'accepted', }) expect(mocks.setEmailMessageClassification).toHaveBeenLastCalledWith({ + env: context.env, db: context.env.APP_DB, userId: 'user-1', messageId: 'message-1', classification: 'accepted', classificationReason: null, }) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ - env: context.env, - db: context.env.APP_DB, - userId: 'user-1', - messageId: 'message-1', - }) mocks.setEmailMessageClassification.mockResolvedValueOnce(false) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() await expect( emailMessageClassifyCapability.handler( { message_id: 'missing', classification: 'quarantined' }, context, ), ).rejects.toThrow('Email message not found: missing') - expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() - - mocks.setEmailMessageClassification.mockResolvedValueOnce(true) - mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ - messageId: 'message-1', - message: { - status: 'error', - error: new Error('mailbox mirror timed out'), - }, - events: [], - }) - await expect( - emailMessageClassifyCapability.handler( - { message_id: 'message-1', classification: 'quarantined' }, - context, - ), - ).resolves.toEqual({ - message_id: 'message-1', - classification: 'quarantined', - }) }) diff --git a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts index 1a9562ddf9..08ec83f2bb 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-classify.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-classify.ts @@ -1,8 +1,7 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' -import { mirrorMailboxMessageGraphFromD1 } from '#worker/email/mailbox-live-mirror.ts' -import { setEmailMessageClassification } from '#worker/email/repo.ts' +import { setEmailMessageClassification } from '#worker/email/service.ts' import { emailClassificationValues } from '#worker/email/types.ts' import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' @@ -36,6 +35,7 @@ export const emailMessageClassifyCapability = defineDomainCapability( const classificationReason = args.classification === 'quarantined' ? 'Reclassified by user.' : null const updated = await setEmailMessageClassification({ + env: ctx.env, db: ctx.env.APP_DB, userId: user.userId, messageId: args.message_id, @@ -45,12 +45,6 @@ export const emailMessageClassifyCapability = defineDomainCapability( if (!updated) { throw new Error(`Email message not found: ${args.message_id}`) } - await mirrorMailboxMessageGraphFromD1({ - env: ctx.env, - db: ctx.env.APP_DB, - userId: user.userId, - messageId: args.message_id, - }) return { message_id: args.message_id, classification: args.classification, From d747c2d4c06d3177fc476892d39dd710d2b1c71f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 11:05:16 +0000 Subject: [PATCH 09/62] fix(email): repair persisted provider mirror gaps Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 74 +++++---- .../src/email/delivery-queue.node.test.ts | 153 +++++++++++++++++- packages/worker/src/email/delivery-queue.ts | 28 ++-- .../email/mailbox-live-mirror.node.test.ts | 49 +++++- .../worker/src/email/mailbox-live-mirror.ts | 42 +++-- packages/worker/src/email/mailbox-types.ts | 2 +- .../outbound-mailbox-mirror.workers.test.ts | 2 + 7 files changed, 284 insertions(+), 66 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 3ec77dc79e..07765dacc0 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -219,11 +219,11 @@ Durable Object export behavior: delivery events) through the account-export `mailbox` section (`exportMailbox` RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires - non-inbound live dual-write for outbound terminals, provider-recorded delivery - events, and user classification (`service.ts#setEmailMessageClassification`) - without changing D1 authority — D1 `email_*` rows remain the live source of - truth and are still exported in the `d1` section. See - [Mailbox](#durable-objects-mailbox). + non-inbound live dual-write for outbound terminals, provider delivery-queue + graph repair, and user classification + (`service.ts#setEmailMessageClassification`) without changing D1 authority — + D1 `email_*` rows remain the live source of truth and are still exported in + the `d1` section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -764,21 +764,26 @@ owner-bound `upsertDeliveryEvents` batch RPC (not concurrent per-event RPCs to the same DO). Event load queries newest `max+1` rows from D1, restores chronological order, and when truncated keeps the newest `mailboxLiveMirrorMaxEvents` (`mailboxUpsertDeliveryEventsMax`, 100) — dropping -oldest overflow. The batch RPC shares one 1s timeout and one -`mailbox_mirror:upsert_delivery_event_batch` telemetry outcome (timeout/error -apply uniformly to per-event summary entries). Analytics Engine budget per graph -call is two writes (1 message outcome + 1 batch outcome). Never throws; returns -a bounded summary. **Live callers (non-inbound phase-2 slice):** +oldest overflow — with a stable warning (`mailbox-live-mirror-events-truncated`; +`userId`, `messageId`, `loaded`, `max`). The batch RPC shares one 1s timeout and +one `mailbox_mirror:upsert_delivery_event_batch` telemetry outcome +(timeout/error apply uniformly to per-event summary entries). Each graph attempt +emits at most two Analytics Engine writes (1 message outcome + 1 batch outcome). +Never throws; returns a bounded summary. **Live callers (non-inbound phase-2 +slice):** - **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full message/thread/attachment/event graph. Passes a just-created thread when present; otherwise the orchestrator loads it from D1. Failures never affect send/refund. -- **Provider-recorded delivery events** (`delivery-queue.ts`) — when queue - ingestion persists a new provider event (`recorded`), schedules - `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack is not - blocked; full graph repair via message + batch event upsert. +- **Provider delivery queue** (`delivery-queue.ts`) — when queue ingestion + resolves to `recorded`, `duplicate`, or `stale` with a bound message, + schedules `mirrorMailboxMessageGraphFromD1` via `waitUntil` so the Worker ack + is not blocked; full graph repair via message + batch event upsert. `recorded` + and `duplicate` still run subscription dispatch first; `stale` skips + subscription dispatch (abuse pause only) but still schedules graph repair when + a message is present. - **User classification** (`service.ts#setEmailMessageClassification`) — transport handlers (`account-email.ts`, `email-message-classify.ts`) delegate here for the D1 mutation + full graph mirror invariant (mirror only after a @@ -810,20 +815,22 @@ mail content. Account export pages Mailbox state through the `mailbox` section This is an expand/contract migration. **Phase 2 is partially wired (non-inbound slice):** live dual-write covers outbound terminal -message/thread/attachment/event graphs, provider-recorded delivery events (queue -`waitUntil` full graph repair), and user classification (full graph repair after -D1 update via `service.ts#setEmailMessageClassification`). Mirror helpers emit -automatic namespaced outcome telemetry (`missing` and `timeout` included). -Message and batch event repair each use one 1s-bounded RPC; graph event repair -batches up to 100 events with explicit truncation (newest retained, -chronological insertion restored) and one batch telemetry outcome. -**`system:email` stays in D1 only.** Inbound lifecycle dual-write, -retention/metadata deletes, and scheduled parity reconcile are **not** wired in -this slice. Live email read/write authority is unchanged — D1 `email_*` tables -remain authoritative for all live paths. D1 email rows and existing R2 inventory -deletion stay authoritative during expand. **Phase 2 completes only when every -user-mail mutation that writes D1 also writes the DO and parity/reconciliation -are wired** — inbound paths and parity reconcile remain. +message/thread/attachment/event graphs, provider delivery-queue graph repair +(`recorded` / `duplicate` / `stale` with a message, via `waitUntil`), and user +classification (full graph repair after D1 update via +`service.ts#setEmailMessageClassification`). Mirror helpers emit automatic +namespaced outcome telemetry (`missing` and `timeout` included). Message and +batch event repair each use one 1s-bounded RPC; each graph attempt emits at most +two AE writes; event repair batches up to 100 events with explicit truncation +(newest retained, chronological insertion restored, stable truncation warning) +and one batch telemetry outcome. **`system:email` stays in D1 only.** Inbound +lifecycle dual-write, retention/metadata deletes, and scheduled parity reconcile +are **not** wired in this slice. Live email read/write authority is unchanged — +D1 `email_*` tables remain authoritative for all live paths. D1 email rows and +existing R2 inventory deletion stay authoritative during expand. **Phase 2 +completes only when every user-mail mutation that writes D1 also writes the DO +and parity/reconciliation are wired** — inbound paths and parity reconcile +remain. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -839,13 +846,14 @@ are wired** — inbound paths and parity reconcile remain. `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, - provider `recorded` queue events (`waitUntil`), and user classification + provider delivery-queue outcomes with a message (`recorded`, `duplicate`, + `stale`; `waitUntil`), and user classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` - outcomes automatically (graph repair uses message RPC + one batch event RPC). - **Still pending for phase-2 completion:** inbound commit/touch/classify - dual-write, retention and metadata-delete mirrors, and scheduled parity - compare/reconcile. + outcomes automatically (graph repair uses message RPC + one batch event RPC; + at most two AE writes per attempt). **Still pending for phase-2 completion:** + inbound commit/touch/classify dual-write, retention and metadata-delete + mirrors, and scheduled parity compare/reconcile. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified. D1 writes continue. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata diff --git a/packages/worker/src/email/delivery-queue.node.test.ts b/packages/worker/src/email/delivery-queue.node.test.ts index 8030c12213..f7a81a109c 100644 --- a/packages/worker/src/email/delivery-queue.node.test.ts +++ b/packages/worker/src/email/delivery-queue.node.test.ts @@ -159,14 +159,26 @@ test('email delivery Queue acknowledges permanent outcomes and retries unmatched waitUntil: expect.any(Function), }, ) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(3) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenNthCalledWith(1, { env: expect.anything(), db: expect.anything(), userId: 'user-1', messageId: 'message-1', }) - expect(waitUntilPromises).toHaveLength(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenNthCalledWith(2, { + env: expect.anything(), + db: expect.anything(), + userId: 'user-1', + messageId: 'message-1', + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenNthCalledWith(3, { + env: expect.anything(), + db: expect.anything(), + userId: 'user-1', + messageId: 'message-1', + }) + expect(waitUntilPromises).toHaveLength(3) expect(consoleWarn).toHaveBeenCalledWith('email-delivery-event-unmatched', { queueMessageId: 'queue-unmatched', providerMessageId: 'provider-1', @@ -290,3 +302,138 @@ test('mailbox graph mirror failures scheduled in waitUntil do not reject or alte 'mailbox rpc unavailable', ) }) + +test('stale inserted provider events schedule mailbox graph mirror without subscription dispatch', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + + const providerEvent = { + payload: { + eventId: 'event-stale-insert', + messageId: 'provider-stale', + delivery: { status: 'deferred' }, + }, + } + const storedMessage = { id: 'message-stale', userId: 'user-stale' } + mocks.processCloudflareEmailDeliveryEvent.mockResolvedValue({ + outcome: 'stale', + providerEvent, + message: storedMessage, + }) + + const queueMessage = createQueueMessage('queue-stale-insert', { + kind: 'stale', + }) + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue( + { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch, + { APP_DB: {} } as Env, + ctx, + ) + await Promise.all(waitUntilPromises) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).not.toHaveBeenCalled() + expect(mocks.dispatchEmailDeliverySubscriptionEvents).not.toHaveBeenCalled() + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: expect.anything(), + db: expect.anything(), + userId: 'user-stale', + messageId: 'message-stale', + }) + expect(waitUntilPromises).toHaveLength(1) +}) + +test('duplicate retry after recorded dispatch failure schedules mailbox graph mirror', async () => { + consoleError.mockImplementation(() => {}) + mocks.processCloudflareEmailDeliveryEvent.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.dispatchEmailDeliverySubscriptionEvents.mockReset() + + const providerEvent = { + payload: { + eventId: 'event-retry', + messageId: 'provider-retry', + delivery: { status: 'delivered' }, + }, + } + const storedMessage = { id: 'message-retry', userId: 'user-retry' } + mocks.processCloudflareEmailDeliveryEvent + .mockResolvedValueOnce({ + outcome: 'recorded', + providerEvent, + event: { id: 'd1-event-retry' }, + message: storedMessage, + }) + .mockResolvedValueOnce({ + outcome: 'duplicate', + providerEvent, + message: storedMessage, + }) + mocks.dispatchEmailDeliverySubscriptionEvents + .mockRejectedValueOnce( + new Error('artifact_preparation_failed: transient KV failure'), + ) + .mockResolvedValueOnce([]) + + const queueMessage = createQueueMessage('queue-retry', { kind: 'recorded' }) + const ctx = { + waitUntil(promise: Promise) { + void promise + }, + passThroughOnException() {}, + } as ExecutionContext + const batch = { + queue: 'kody-email-delivery', + messages: [queueMessage], + ackAll() {}, + retryAll() {}, + } as unknown as MessageBatch + + await handleEmailDeliveryQueue(batch, { APP_DB: {} } as Env, ctx) + expect(queueMessage.ack).not.toHaveBeenCalled() + expect(queueMessage.retry).toHaveBeenCalledWith({ delaySeconds: 30 }) + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + + const waitUntilPromises: Array> = [] + const retryCtx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + + await handleEmailDeliveryQueue(batch, { APP_DB: {} } as Env, retryCtx) + await Promise.all(waitUntilPromises) + + expect(queueMessage.ack).toHaveBeenCalledTimes(1) + expect(queueMessage.retry).toHaveBeenCalledTimes(1) + expect(mocks.dispatchEmailDeliverySubscriptionEvents).toHaveBeenCalledTimes(2) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith({ + env: expect.anything(), + db: expect.anything(), + userId: 'user-retry', + messageId: 'message-retry', + }) + expect(waitUntilPromises).toHaveLength(1) + expect(consoleError).toHaveBeenCalledWith( + 'email-delivery-event-processing-failed', + expect.objectContaining({ message: expect.stringContaining('KV failure') }), + ) +}) diff --git a/packages/worker/src/email/delivery-queue.ts b/packages/worker/src/email/delivery-queue.ts index bb843ae987..ce9f7b8ff4 100644 --- a/packages/worker/src/email/delivery-queue.ts +++ b/packages/worker/src/email/delivery-queue.ts @@ -7,6 +7,22 @@ import { type EmailReportingEnv } from './reporting-events.ts' const unmatchedRetryDelaySeconds = 30 export const emailDeliveryQueueName = 'kody-email-delivery' +function scheduleMailboxGraphMirror( + waitUntil: (promise: Promise) => void, + env: Env, + message: { userId: string; id: string } | null, +) { + if (message == null) return + waitUntil( + mirrorMailboxMessageGraphFromD1({ + env, + db: env.APP_DB, + userId: message.userId, + messageId: message.id, + }), + ) +} + export async function handleEmailDeliveryQueue( batch: MessageBatch, env: Env, @@ -43,6 +59,7 @@ export async function handleEmailDeliveryQueue( deliveryStatus: result.providerEvent.payload.delivery.status, eventRecorded: false, }) + scheduleMailboxGraphMirror(waitUntil, env, result.message) queueMessage.ack() break case 'duplicate': @@ -63,16 +80,7 @@ export async function handleEmailDeliveryQueue( providerEvent: result.providerEvent, waitUntil, }) - if (result.outcome === 'recorded' && result.message != null) { - waitUntil( - mirrorMailboxMessageGraphFromD1({ - env, - db: env.APP_DB, - userId: result.message.userId, - messageId: result.message.id, - }), - ) - } + scheduleMailboxGraphMirror(waitUntil, env, result.message) queueMessage.ack() break } diff --git a/packages/worker/src/email/mailbox-live-mirror.node.test.ts b/packages/worker/src/email/mailbox-live-mirror.node.test.ts index 91be2c89f8..f9109c71df 100644 --- a/packages/worker/src/email/mailbox-live-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-live-mirror.node.test.ts @@ -136,9 +136,9 @@ async function seedMessageGraph(db: D1Database) { }) } -test('live-mirror AE write budget stays under Analytics Engine request cap', () => { - expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) +test('live-mirror AE write budget is at most two writes per graph attempt', () => { expect(mailboxLiveMirrorMaxAnalyticsWrites).toBe(2) + expect(mailboxLiveMirrorMaxAnalyticsWrites).toBeLessThan(250) }) test('live-mirror graph mirrors message before one batch event RPC and uses createdAt for event updatedAt', async () => { @@ -316,6 +316,7 @@ test('live-mirror graph uses one batch RPC after message settles, not per-event }) test('live-mirror graph keeps newest events when truncated and sets eventsTruncated', async () => { + consoleWarn.mockImplementation(() => {}) const { db } = await createEmailDb() await seedMessage(db) @@ -369,7 +370,7 @@ test('live-mirror graph keeps newest events when truncated and sets eventsTrunca expect(batchEvents.at(-1)?.id).toBe( `evt-${String(mailboxLiveMirrorMaxEvents).padStart(3, '0')}`, ) - // 1 message + 1 batch AE write; never per-event fan-out. + // At most two AE writes: message + batch; never per-event fan-out. expect(writeDataPoint).toHaveBeenCalledTimes( mailboxLiveMirrorMaxAnalyticsWrites, ) @@ -381,6 +382,15 @@ test('live-mirror graph keeps newest events when truncated and sets eventsTrunca 'mailbox_mirror:upsert_delivery_event_batch', ), ).toBe(true) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-live-mirror-events-truncated', + { + userId: 'user-aaa', + messageId: 'msg-1', + loaded: totalEvents, + max: mailboxLiveMirrorMaxEvents, + }, + ) }) test('live-mirror graph maps batch timeout and error onto each event summary', async () => { @@ -476,7 +486,8 @@ test('live-mirror graph returns missing when D1 message is absent', async () => ok: true as const, accepted: true, })) - const { env } = fakeMailboxEnv({ mirrorMessage }) + const upsertDeliveryEvents = vi.fn() + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvents }) await expect( mirrorMailboxMessageGraphFromD1({ @@ -492,6 +503,36 @@ test('live-mirror graph returns missing when D1 message is absent', async () => eventsTruncated: false, }) expect(mirrorMessage).not.toHaveBeenCalled() + expect(upsertDeliveryEvents).not.toHaveBeenCalled() +}) + +test('live-mirror graph returns missing for cross-owner message without RPCs', async () => { + const { db } = await createEmailDb() + await seedMessageGraph(db) + const mirrorMessage = vi.fn(async () => ({ + ok: true as const, + accepted: true, + })) + const upsertDeliveryEvents = vi.fn(async () => ({ + results: [], + })) + const { env } = fakeMailboxEnv({ mirrorMessage, upsertDeliveryEvents }) + + await expect( + mirrorMailboxMessageGraphFromD1({ + env, + db, + userId: 'other-user', + messageId: 'msg-1', + }), + ).resolves.toEqual({ + messageId: 'msg-1', + message: { status: 'missing' }, + events: [], + eventsTruncated: false, + }) + expect(mirrorMessage).not.toHaveBeenCalled() + expect(upsertDeliveryEvents).not.toHaveBeenCalled() }) test('live-mirror graph catches unexpected errors without throwing', async () => { diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts index 284f20d7fc..493a6c0e66 100644 --- a/packages/worker/src/email/mailbox-live-mirror.ts +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -29,8 +29,8 @@ import { type EmailThreadRecord } from './types.ts' * round-trip; otherwise the graph helper loads it with `getEmailThreadById`. * * Graph event mirroring uses one batch RPC after the message settles (not - * Promise.all of per-event RPCs). Analytics Engine parity writes stay small: - * 1 message + 1 batch outcome. + * Promise.all of per-event RPCs). Analytics Engine parity writes are at most + * two per graph attempt: 1 message outcome + 1 batch outcome. */ /** @@ -39,7 +39,7 @@ import { type EmailThreadRecord } from './types.ts' */ export const mailboxLiveMirrorMaxEvents = mailboxUpsertDeliveryEventsMax -/** AE write budget: message outcome + one batch outcome. */ +/** AE write budget cap: at most two writes (message + batch). */ export const mailboxLiveMirrorMaxAnalyticsWrites = 2 export type MailboxLiveMirrorEnv = MailboxMirrorEnv @@ -143,19 +143,23 @@ export async function mirrorMailboxMessageGraphFromD1(input: { return emptyGraphSummary(input.messageId, { status: 'missing' }) } - const attachments = await listEmailAttachmentsForMessage({ - db: input.db, - messageId: input.messageId, - }) - - let thread = input.thread - if (thread === undefined && message.threadId) { - thread = await getEmailThreadById({ + // After the owner-scoped message load, fan out attachment + optional + // thread reads concurrently. + const [attachments, thread] = await Promise.all([ + listEmailAttachmentsForMessage({ db: input.db, - userId: input.userId, - threadId: message.threadId, - }) - } + messageId: input.messageId, + }), + input.thread !== undefined + ? Promise.resolve(input.thread) + : message.threadId + ? getEmailThreadById({ + db: input.db, + userId: input.userId, + threadId: message.threadId, + }) + : Promise.resolve(null), + ]) // Message settles before the event-batch RPC starts (ordering guarantee). const messageResult = await mirrorMailboxMessageSnapshot({ @@ -174,6 +178,14 @@ export async function mirrorMailboxMessageGraphFromD1(input: { limit: mailboxLiveMirrorMaxEvents + 1, }) const eventsTruncated = loadedEvents.length > mailboxLiveMirrorMaxEvents + if (eventsTruncated) { + console.warn('mailbox-live-mirror-events-truncated', { + userId: input.userId, + messageId: input.messageId, + loaded: loadedEvents.length, + max: mailboxLiveMirrorMaxEvents, + }) + } const eventInputs = eventsTruncated ? loadedEvents.slice(-mailboxLiveMirrorMaxEvents) : loadedEvents diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index f506665e2d..5999409de4 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -21,7 +21,7 @@ export const mailboxDeliveryEventRetentionDays = 90 export const mailboxDefaultPageSize = 100 export const mailboxMaxPageSize = 500 /** Max complete delivery-event snapshots per `upsertDeliveryEvents` RPC. */ -export const mailboxUpsertDeliveryEventsMax = mailboxDefaultPageSize +export const mailboxUpsertDeliveryEventsMax = 100 export const mailboxRetentionBatchSize = 100 export const mailboxRetentionAlarmSkewMs = 60_000 /** Bound retry delay when overdue retention work cannot finish (e.g. R2 errors). */ diff --git a/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts index c93610214b..c62fa0c27a 100644 --- a/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts +++ b/packages/worker/src/email/outbound-mailbox-mirror.workers.test.ts @@ -32,6 +32,7 @@ function createMailboxStubEnv(input: { const stub = { mirrorMessage: method, upsertDeliveryEvent: method, + upsertDeliveryEvents: method, touchThread: method, updateMessageDelivery: method, setMessageClassification: method, @@ -207,6 +208,7 @@ test('sendOutboundEmail ignores Mailbox RPC failure and timeout', async () => { silenceIncidentalRuntimeWarnings([ 'mailbox-mirror-message-failed', 'mailbox-mirror-delivery-event-failed', + 'mailbox-mirror-delivery-event-batch-failed', 'mailbox-live-mirror-message-graph-failed', ]) await ensureEmailTestSchema(env.APP_DB) From 0551015dbf6d8b5e8e6bb708f1154bd3149f7ad6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 11:45:40 +0000 Subject: [PATCH 10/62] feat(email): add Mailbox parity soak gate Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 157 +++- .../architecture/feature-flags.md | 32 + .../migrations/0125-mailbox-parity-state.sql | 34 + .../handlers/admin-feature-flags.node.test.ts | 66 +- .../app/handlers/session-handler.node.test.ts | 1 + .../worker/src/email/mailbox-parity-repo.ts | 372 +++++++++ .../email/mailbox-read-cutover.node.test.ts | 528 ++++++++++++ .../worker/src/email/mailbox-read-cutover.ts | 195 +++++ .../src/email/mailbox-reconcile.node.test.ts | 783 +++++++++++++++++ .../worker/src/email/mailbox-reconcile.ts | 787 ++++++++++++++++++ .../email/mailbox-reconcile.workers.test.ts | 331 ++++++++ .../src/feature-flags/registry.node.test.ts | 26 + packages/worker/src/feature-flags/registry.ts | 13 + .../src/feature-flags/service.node.test.ts | 21 +- .../capabilities/access-control.node.test.ts | 1 + ...min-feature-flag-capabilities.node.test.ts | 38 +- .../scheduled/scheduled-lanes.node.test.ts | 54 ++ .../worker/src/scheduled/scheduled-lanes.ts | 15 + packages/worker/src/users-test-schema.ts | 16 + 19 files changed, 3399 insertions(+), 71 deletions(-) create mode 100644 packages/worker/migrations/0125-mailbox-parity-state.sql create mode 100644 packages/worker/src/email/mailbox-parity-repo.ts create mode 100644 packages/worker/src/email/mailbox-read-cutover.node.test.ts create mode 100644 packages/worker/src/email/mailbox-read-cutover.ts create mode 100644 packages/worker/src/email/mailbox-reconcile.node.test.ts create mode 100644 packages/worker/src/email/mailbox-reconcile.ts create mode 100644 packages/worker/src/email/mailbox-reconcile.workers.test.ts create mode 100644 packages/worker/src/feature-flags/registry.node.test.ts create mode 100644 packages/worker/src/scheduled/scheduled-lanes.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 07765dacc0..e8d9a38814 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -268,11 +268,19 @@ The schema is defined by migrations in `packages/worker/migrations/`: D1 payload storage-byte read, enforcement, and reconciliation. UserMeter `storage_bytes_state` (schema v4) and `package_service_states` (schema v5) are optional expand-phase shadows only — see - [Entitlements](./entitlements.md#usermeter-expand-phase). Inbound email - routing does not reverse-resolve stable ids at all — it uses the indexed - username lookup (`findPublicUserIdentityByUsername`). Contextless paths - resolve stable ids with one indexed point read on `users.stable_user_id` (for - example `findUserAccountByStableUserId`). + [Entitlements](./entitlements.md#usermeter-expand-phase). Mailbox expand-phase + parity/backfill state (migration `0125-mailbox-parity-state.sql`): + `mailbox_parity_checked_at`, `mailbox_parity_matching_since`, + `mailbox_parity_mismatch_count`, `mailbox_parity_last_error`, + `mailbox_parity_content_watermark_at`, message and orphan-event backfill + cursors/completion timestamps on the user row. Indexed by + `(mailbox_parity_checked_at, stable_user_id)` for oldest-first discovery in + the hourly `mailbox_parity` reconcile lane (see + [Mailbox](#durable-objects-mailbox)). Inbound email routing does not + reverse-resolve stable ids at all — it uses the indexed username lookup + (`findPublicUserIdentityByUsername`). Contextless paths resolve stable ids + with one indexed point read on `users.stable_user_id` (for example + `findUserAccountByStableUserId`). - `platform_feedback`: attributed, user-approved Kody feedback and admin triage state. Submitter identity remains on the row; optional reviewer attribution is cleared if that admin account is deleted. Open and triaged rows remain until @@ -382,8 +390,10 @@ delivery-event table because Wrangler's emulated dataset has no SQL API. with a separate row shape defined in `packages/worker/src/email/mailbox-parity-events.ts` (`recordMailboxParityEvent`). Live mirror helpers record one namespaced outcome -per attempted operation; scheduled parity compares are not wired yet. These rows -do not feed the admin insights charts today: +per attempted operation; the hourly `mailbox_parity` reconcile lane records one +outcome per count comparison (`compare_threads`, `compare_messages`, +`compare_attachments`, `compare_delivery_events`). These rows do not feed the +admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) - `blob1` — namespaced event type (`mailbox_mirror:` or @@ -402,9 +412,9 @@ Mirror operations: `mirror_message`, `upsert_delivery_event`, `stale`, `missing`, `timeout`, `skipped`, `error`. Parity operations: `compare_threads`, `compare_messages`, `compare_attachments`, `compare_delivery_events`. Parity outcomes: `match`, `mismatch`. Mirror outcome -writes are automatic from live dual-write helpers; parity compare writers remain -for future reconciliation. Writes are best-effort and never throw into D1 -authority paths. +writes are automatic from live dual-write helpers; parity compare writes come +from the scheduled reconcile lane after owner-scoped D1 and Mailbox counts are +compared. Writes are best-effort and never throw into D1 authority paths. Two D1 reporting projections deliberately remain: @@ -790,8 +800,69 @@ slice):** successful D1 update; failures never change the mutation response). Inbound commit/touch/classify paths, retention deletes, and partial mutation -mirrors are **not** wired in this slice. Parity compare/reconcile cron remains -future work. +mirrors are **not** wired in this slice. Scheduled parity reconcile **is** wired +(see below); read cutover is prepared but not flipped. + +**Scheduled parity reconcile (`mailbox_parity`)** runs as its own queue-isolated +scheduled lane (`packages/worker/src/scheduled/scheduled-lanes.ts`, +`packages/worker/src/email/mailbox-reconcile.ts`). The five-minute Worker cron +dispatches it only on the top of each UTC hour (`shouldRunMailboxParityCron`); +production uses the same per-lane queue isolation as reconcile, retention, and +sibling lanes so a slow parity pass cannot consume another lane's budget. + +User discovery is **D1-only** — no Mailbox Durable Object enumeration. Each tick +selects up to eight non-deleting mail owners (`mailboxParityUserBatchSize`) +ordered by oldest `mailbox_parity_checked_at`, excluding `system:email`, who +have at least one `email_messages` or `email_delivery_events` row. Per-user work +is bounded by a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`). + +Per user, the lane: + +1. **Content watermark baseline** — on the first backfill attempt, sets + `mailbox_parity_content_watermark_at` to the run's `now` and retains it + across incomplete/error ticks so updates during creation/orphan mirroring + stay inside the replay window. +2. **Bounded initial message backfill** — keyset-pages D1 `email_messages` by + `(created_at, id)` and mirrors each graph via + `mirrorMailboxMessageGraphFromD1` until complete or budget-exhausted. +3. **Bounded orphan-event backfill** — after messages complete, keyset-pages D1 + `email_delivery_events` with `message_id IS NULL` and mirrors each via + `mirrorMailboxOrphanDeliveryEventFromD1`. +4. **Content watermark replay** — after both initial backfills complete, + keyset-replays every owner message with `updated_at` in + `(watermark, current-now]` (classification-like updates that landed after a + graph mirror but before backfill finished). The watermark advances only when + the full window succeeds. +5. **Count compare + AE signals** — owner-scoped D1 counts vs + `Mailbox.countMailbox()` for threads, messages, attachments, and delivery + events. Each comparison emits one `mailbox_parity:` Analytics + Engine row with `double2 = d1Count - doCount`. + +Parity progress persists on `users` (migration `0125-mailbox-parity-state.sql`). +D1 remains mail authority — the lane never mutates `email_*` rows and does not +flip read authority. + +**`matching_since` semantics:** records the start of continuous **exact** +D1↔Mailbox count parity for phase-3 soak tracking. Set on the first exact +compare when null; preserved across successful content replay + exact compare. +Cleared on: initial backfill work in the same tick (`initialBackfilled > 0`), +incomplete/budget-exhausted ticks, mirror retryable failures, content-replay +failure/incomplete, compare mismatch (also increments consecutive +`mailbox_parity_mismatch_count` and reopens bounded creation/orphan backfill +while retaining the content watermark), and compare-path errors +(`rotateCheckedAt` also clears soak so a poison tick cannot preserve a false +window). Resets `mismatch_count` to `0` on exact match. + +**Phase 3 read cutover (prepared, not flipped):** +`packages/worker/src/email/mailbox-read-cutover.ts` defines a default-off +`mailbox-read-cutover` feature flag plus `getOwnerEmailMessageById` / +`isMailboxReadCutoverEnabled`. The gate requires the flag **and** per-user +parity soak: `mailbox_parity_matching_since` at least 24h old, +`mailbox_parity_checked_at` fresh within 2h, +`mailbox_parity_mismatch_count === 0`, and exact `stable_user_id` match. D1 +errors, missing parity state, and an off flag fail closed to D1 reads. No live +reader calls this adapter yet; exposure recording is intentionally omitted until +a live gate site exists (see [Feature flags](./feature-flags.md)). **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -823,14 +894,16 @@ namespaced outcome telemetry (`missing` and `timeout` included). Message and batch event repair each use one 1s-bounded RPC; each graph attempt emits at most two AE writes; event repair batches up to 100 events with explicit truncation (newest retained, chronological insertion restored, stable truncation warning) -and one batch telemetry outcome. **`system:email` stays in D1 only.** Inbound -lifecycle dual-write, retention/metadata deletes, and scheduled parity reconcile -are **not** wired in this slice. Live email read/write authority is unchanged — -D1 `email_*` tables remain authoritative for all live paths. D1 email rows and -existing R2 inventory deletion stay authoritative during expand. **Phase 2 -completes only when every user-mail mutation that writes D1 also writes the DO -and parity/reconciliation are wired** — inbound paths and parity reconcile -remain. +and one batch telemetry outcome. **`system:email` stays in D1 only.** The hourly +queue-isolated `mailbox_parity` lane backfills historical graphs, replays +content-watermark updates, compares owner-scoped D1 vs Mailbox counts, and +persists soak state on `users` (migration `0125`). Inbound lifecycle dual-write, +retention/metadata deletes, and live read cutover are **still not wired**. Live +email read/write authority is unchanged — D1 `email_*` tables remain +authoritative for all live paths. D1 email rows and existing R2 inventory +deletion stay authoritative during expand. **Phase 2 completes only when every +user-mail mutation that writes D1 also writes the DO** — inbound paths and +delete mirrors remain pending. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -845,25 +918,35 @@ remain. `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and - `mailbox-parity-events.ts`. **Live today (partial):** outbound terminals, + `mailbox-parity-events.ts`, `mailbox-parity-repo.ts`, and + `mailbox-reconcile.ts`. **Live today (partial):** outbound terminals, provider delivery-queue outcomes with a message (`recorded`, `duplicate`, `stale`; `waitUntil`), and user classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` outcomes automatically (graph repair uses message RPC + one batch event RPC; - at most two AE writes per attempt). **Still pending for phase-2 completion:** - inbound commit/touch/classify dual-write, retention and metadata-delete - mirrors, and scheduled parity compare/reconcile. + at most two AE writes per attempt). The hourly `mailbox_parity` scheduled + lane backfills, content-watermark replays, count compares, and soak tracking + on `users` (migration `0125`). **Still pending for phase-2 completion:** + inbound commit/touch/classify dual-write and retention/metadata-delete + mirrors. 3. **Reads cut over after production soak** — user-mail reads move to the DO - only after production parity soak is verified. D1 writes continue. + only after production parity soak is verified + (`mailbox_parity_matching_since` ≥ 24h continuous exact counts, fresh + `mailbox_parity_checked_at`, zero `mailbox_parity_mismatch_count`) **and** + the default-off `mailbox-read-cutover` flag is enabled per user. D1 writes + continue. A prepared `getOwnerEmailMessageById` adapter exists but is **not + wired**; no live reader has flipped and no cutover exposures are recorded + yet. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata to D1; retire dual-write and event/mirror machinery used only for the migration. 5. **Later contract migrations** — drop retired D1 user-mail tables/columns only after verification. No premature schema deletion. -A cron sibling track owns scheduled-lane wiring (for example parity / -reconciliation ticks). This Mailbox track owns DO storage semantics, alarm +The hourly `mailbox_parity` scheduled lane (queue-isolated sibling in +`scheduled-lanes.ts`) owns backfill, content-watermark replay, count parity, and +soak state on `users`. This Mailbox track owns DO storage semantics, alarm retention, and the inbound durability boundary below. ### What stays in D1 @@ -1156,15 +1239,17 @@ Artifacts tokens. Reconcile runs through the registry in `packages/worker/src/scheduled/scheduled-lanes.ts`, alongside repo-session -cleanup, system-email retention, general retention, job retention, and hourly -usage-rollup aggregation. Each production queue message preserves the existing -`scheduled_lane_failed` / D1 lock-contention log and Sentry context. A handled -lane failure is acknowledged and retried by the next cron tick, matching the old -cron semantics. A failed enqueue is reported and runs through the inline -fallback after all sibling enqueue attempts finish; multiple failed enqueues -fall back sequentially to avoid D1 lock contention. Consumer transport failures -retain the configured retry/DLQ behavior. No failure can abort or mask a sibling -invocation. +cleanup, system-email retention, general retention, job retention, hourly +usage-rollup aggregation, and the hourly `mailbox_parity` lane (top of each UTC +hour only; D1 user discovery, bounded backfill, content-watermark replay, and +count compare — see [Mailbox](#durable-objects-mailbox)). Each production queue +message preserves the existing `scheduled_lane_failed` / D1 lock-contention log +and Sentry context. A handled lane failure is acknowledged and retried by the +next cron tick, matching the old cron semantics. A failed enqueue is reported +and runs through the inline fallback after all sibling enqueue attempts finish; +multiple failed enqueues fall back sequentially to avoid D1 lock contention. +Consumer transport failures retain the configured retry/DLQ behavior. No failure +can abort or mask a sibling invocation. Production note: diff --git a/docs/contributing/architecture/feature-flags.md b/docs/contributing/architecture/feature-flags.md index 28cf7fcef5..627b59b25e 100644 --- a/docs/contributing/architecture/feature-flags.md +++ b/docs/contributing/architecture/feature-flags.md @@ -69,6 +69,38 @@ The registry ships with one permanent flag, `demo-indicator`, which renders a small badge in the app chrome and exists so the system stays exercised end-to-end (`e2e/admin-feature-flags.spec.ts`). +### Mailbox read cutover (`mailbox-read-cutover`) + +Default-off migration flag for phase 3 of the Mailbox expand/contract track (see +[Data storage — Mailbox](./data-storage.md#durable-objects-mailbox)). It gates +whether owner-scoped user-mail **message reads** may come from the per-user +Mailbox Durable Object instead of D1. + +Evaluation alone is not enough. A live read path must also pass the per-user +parity soak on `users` (migration `0125-mailbox-parity-state.sql`): + +- `mailbox_parity_matching_since` at least **24 hours** old (continuous exact + D1↔Mailbox count parity), +- `mailbox_parity_checked_at` fresh within **2 hours** (hourly lane + slack), +- `mailbox_parity_mismatch_count === 0`, +- exact `stable_user_id` match between the gate caller and the loaded row. + +D1 errors, missing/incomplete parity state, stale `checked_at`, non-zero +mismatch depth, and an off flag all **fail closed** to D1-authoritative reads. +When the gate is on, Mailbox DO read errors propagate with no D1 fallback. + +The prepared adapter lives in +`packages/worker/src/email/mailbox-read-cutover.ts` +(`isMailboxReadCutoverEnabled`, `getOwnerEmailMessageById`). **No live reader +calls it yet** — phase 3 is not flipped. Exposure recording is intentionally +omitted until a live gate site exists (the session/MCP evaluation caches, or an +explicit `recordFeatureFlagExposures` at the first switched call site). Until +then, the flag appears in admin/session evaluation maps but records no +`FLAG_EXPOSURES` rows. + +Success metric: `email_received` error rate should decrease after verified +parity soak and cutover (`successMetric` on the registry definition). + ## Success metrics Every flag exists to move something; the `successMetric` field on a registry diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql new file mode 100644 index 0000000000..3a2af01586 --- /dev/null +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -0,0 +1,34 @@ +-- Account-owned Mailbox parity / backfill state for the hourly reconcile lane. +-- Low-write columns live on users (not per-message). Creation cursors are +-- keyset (created_at, id) so message and message_id-null orphan-event backfill +-- can resume across ticks without DO enumeration. content_watermark_at is set +-- to the run's now at the start of the first backfill attempt (retained across +-- incomplete/error ticks) and bounds a keyset replay of messages with +-- updated_at in (watermark, current-now] after both initial backfills complete +-- so classification-like updates — including those that land after a graph was +-- mirrored but before backfill finished — are remirrored even when counts stay +-- equal; the watermark advances only after the full window succeeds. +-- matching_since records the start of continuous exact D1 ↔Mailbox count +-- parity for soak tracking; it clears on initial backfill work, +-- incomplete/budget-exhausted ticks, mirror errors, content-replay +-- failure/incomplete, or compare mismatch — but may persist across a +-- successful content replay + exact compare. mismatch_count is consecutive +-- current mismatch depth and resets to 0 on exact match. + +ALTER TABLE users ADD COLUMN mailbox_parity_checked_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_matching_since TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_mismatch_count INTEGER NOT NULL DEFAULT 0 + CHECK (mailbox_parity_mismatch_count >= 0); +ALTER TABLE users ADD COLUMN mailbox_parity_last_error TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_watermark_at TEXT; + +ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_created_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_id TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_completed_at TEXT; + +ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_created_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_id TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_completed_at TEXT; + +CREATE INDEX idx_users_mailbox_parity_checked + ON users(mailbox_parity_checked_at, stable_user_id); diff --git a/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts b/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts index 9f21d66586..5cd10a41c7 100644 --- a/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts +++ b/packages/worker/src/app/handlers/admin-feature-flags.node.test.ts @@ -361,18 +361,30 @@ test('admin feature flags HTTP lifecycle: auth, list, set_global, and validation const listResponse = await handler.handler(createHandlerRequest()) expect(listResponse.status).toBe(200) - await expect(listResponse.json()).resolves.toMatchObject({ - ok: true, - featureFlags: [ - { + const listBody = (await listResponse.json()) as { + ok: boolean + featureFlags: Array<{ key: string }> + } + expect(listBody).toMatchObject({ ok: true }) + expect(listBody.featureFlags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'demo-indicator', stale: false, defaultEnabled: false, global: null, overrides: [], - }, - ], - }) + }), + expect.objectContaining({ + key: 'mailbox-read-cutover', + stale: false, + defaultEnabled: false, + successMetric: expect.objectContaining({ + eventType: 'email_received', + }), + }), + ]), + ) const setGlobalResponse = await handler.handler( createHandlerRequest({ @@ -387,20 +399,24 @@ test('admin feature flags HTTP lifecycle: auth, list, set_global, and validation }), ) expect(setGlobalResponse.status).toBe(200) - await expect(setGlobalResponse.json()).resolves.toMatchObject({ - ok: true, - featureFlags: [ - { + const setGlobalBody = (await setGlobalResponse.json()) as { + ok: boolean + featureFlags: Array<{ key: string }> + } + expect(setGlobalBody).toMatchObject({ ok: true }) + expect(setGlobalBody.featureFlags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'demo-indicator', - global: { + global: expect.objectContaining({ enabled: true, rolloutPercent: 25, note: 'canary', updatedByStableUserId: null, - }, - }, - ], - }) + }), + }), + ]), + ) expect(logAuditEventSpy).toHaveBeenCalledWith( expect.objectContaining({ category: 'admin', @@ -523,10 +539,14 @@ test('admin feature flags set_user_override validates user identity and existenc }), ) expect(byUsername.status).toBe(200) - await expect(byUsername.json()).resolves.toMatchObject({ - ok: true, - featureFlags: [ - { + const byUsernameBody = (await byUsername.json()) as { + ok: boolean + featureFlags: Array<{ key: string }> + } + expect(byUsernameBody).toMatchObject({ ok: true }) + expect(byUsernameBody.featureFlags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'demo-indicator', overrides: [ expect.objectContaining({ @@ -535,7 +555,7 @@ test('admin feature flags set_user_override validates user identity and existenc enabled: true, }), ], - }, - ], - }) + }), + ]), + ) }) diff --git a/packages/worker/src/app/handlers/session-handler.node.test.ts b/packages/worker/src/app/handlers/session-handler.node.test.ts index 213f4b8e4d..a8af528add 100644 --- a/packages/worker/src/app/handlers/session-handler.node.test.ts +++ b/packages/worker/src/app/handlers/session-handler.node.test.ts @@ -167,6 +167,7 @@ test('session handler only renews remembered sessions after the renewal window', permissions: [], featureFlags: { 'demo-indicator': false, + 'mailbox-read-cutover': false, }, }, }) diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts new file mode 100644 index 0000000000..f41c43a582 --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -0,0 +1,372 @@ +import { systemEmailOwnerId } from './email-owner.ts' +import { type MailboxCountResult } from './mailbox-types.ts' + +/** + * D1 loaders/persisters for the hourly Mailbox parity reconcile lane. + * Creation backfill uses (created_at, id) keysets; content replay uses + * (updated_at, id) within (watermark, now]. + */ + +export type MailboxParityBackfillCursor = { + createdAt: string + id: string +} + +export type MailboxParityContentCursor = { + updatedAt: string + id: string +} + +export type MailboxParityUserState = { + userId: string + matchingSince: string | null + mismatchCount: number + contentWatermarkAt: string | null + messageCursor: MailboxParityBackfillCursor | null + messagesCompletedAt: string | null + orphanEventCursor: MailboxParityBackfillCursor | null + orphanEventsCompletedAt: string | null +} + +export type MailboxParityCreatedKeysetRow = { + id: string + created_at: string +} + +export type MailboxParityUpdatedKeysetRow = { + id: string + updated_at: string +} + +export async function listUsersForMailboxParity(input: { + db: D1Database + limit: number +}): Promise> { + const result = await input.db + .prepare( + `SELECT u.stable_user_id AS userId + FROM users u + WHERE u.deleting_at IS NULL + AND u.stable_user_id != ? + AND ( + EXISTS ( + SELECT 1 FROM email_messages m + WHERE m.user_id = u.stable_user_id + ) + OR EXISTS ( + SELECT 1 FROM email_delivery_events e + WHERE e.user_id = u.stable_user_id + ) + ) + ORDER BY u.mailbox_parity_checked_at ASC, u.stable_user_id ASC + LIMIT ?`, + ) + .bind(systemEmailOwnerId, input.limit) + .all<{ userId: string }>() + return result.results ?? [] +} + +export async function countD1MailboxParity(input: { + db: D1Database + userId: string +}): Promise { + const [threads, messages, attachments, deliveryEvents] = await Promise.all([ + input.db + .prepare(`SELECT COUNT(*) AS n FROM email_threads WHERE user_id = ?`) + .bind(input.userId) + .first<{ n: number }>(), + input.db + .prepare(`SELECT COUNT(*) AS n FROM email_messages WHERE user_id = ?`) + .bind(input.userId) + .first<{ n: number }>(), + input.db + .prepare( + `SELECT COUNT(*) AS n + FROM email_attachments ea + JOIN email_messages em ON em.id = ea.message_id + WHERE em.user_id = ?`, + ) + .bind(input.userId) + .first<{ n: number }>(), + input.db + .prepare( + `SELECT COUNT(*) AS n FROM email_delivery_events WHERE user_id = ?`, + ) + .bind(input.userId) + .first<{ n: number }>(), + ]) + return { + threads: Number(threads?.n ?? 0) || 0, + messages: Number(messages?.n ?? 0) || 0, + attachments: Number(attachments?.n ?? 0) || 0, + deliveryEvents: Number(deliveryEvents?.n ?? 0) || 0, + } +} + +export async function loadUserParityState(input: { + db: D1Database + userId: string +}): Promise { + const row = await input.db + .prepare( + `SELECT + stable_user_id AS userId, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, + mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, + mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + FROM users + WHERE stable_user_id = ? + AND deleting_at IS NULL + LIMIT 1`, + ) + .bind(input.userId) + .first<{ + userId: string + matchingSince: string | null + mismatchCount: number | null + contentWatermarkAt: string | null + messageCursorCreatedAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + orphanCursorCreatedAt: string | null + orphanCursorId: string | null + orphanEventsCompletedAt: string | null + }>() + if (!row) return null + const messageCursor = + row.messageCursorCreatedAt != null && row.messageCursorId != null + ? { createdAt: row.messageCursorCreatedAt, id: row.messageCursorId } + : null + const orphanEventCursor = + row.orphanCursorCreatedAt != null && row.orphanCursorId != null + ? { createdAt: row.orphanCursorCreatedAt, id: row.orphanCursorId } + : null + return { + userId: row.userId, + matchingSince: row.matchingSince, + mismatchCount: Number(row.mismatchCount ?? 0) || 0, + contentWatermarkAt: row.contentWatermarkAt, + messageCursor, + messagesCompletedAt: row.messagesCompletedAt, + orphanEventCursor, + orphanEventsCompletedAt: row.orphanEventsCompletedAt, + } +} + +export async function listMessageBackfillPage(input: { + db: D1Database + userId: string + cursor: MailboxParityBackfillCursor | null + limit: number +}): Promise> { + const result = + input.cursor == null + ? await input.db + .prepare( + `SELECT id, created_at + FROM email_messages + WHERE user_id = ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.userId, input.limit) + .all() + : await input.db + .prepare( + `SELECT id, created_at + FROM email_messages + WHERE user_id = ? + AND ( + created_at > ? + OR (created_at = ? AND id > ?) + ) + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.cursor.createdAt, + input.cursor.createdAt, + input.cursor.id, + input.limit, + ) + .all() + return result.results ?? [] +} + +/** + * Cohesive D1 loader for owner-scoped delivery events with `message_id` null + * (orphan / unbound ledger rows). Keyset-paged for bounded backfill. + */ +export async function listOrphanDeliveryEventBackfillPage(input: { + db: D1Database + userId: string + cursor: MailboxParityBackfillCursor | null + limit: number +}): Promise> { + const result = + input.cursor == null + ? await input.db + .prepare( + `SELECT id, created_at + FROM email_delivery_events + WHERE user_id = ? + AND message_id IS NULL + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.userId, input.limit) + .all() + : await input.db + .prepare( + `SELECT id, created_at + FROM email_delivery_events + WHERE user_id = ? + AND message_id IS NULL + AND ( + created_at > ? + OR (created_at = ? AND id > ?) + ) + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.cursor.createdAt, + input.cursor.createdAt, + input.cursor.id, + input.limit, + ) + .all() + return result.results ?? [] +} + +/** + * Keyset page of owner messages updated after `watermarkAt` and at or before + * `upperBoundAt` (run now, or wall time when completing in the baseline run). + * In-tick `cursor` continues within the same window without skipping equal + * `updated_at` rows. + */ +export async function listContentReplayPage(input: { + db: D1Database + userId: string + watermarkAt: string + upperBoundAt: string + cursor: MailboxParityContentCursor | null + limit: number +}): Promise> { + const result = + input.cursor == null + ? await input.db + .prepare( + `SELECT id, updated_at + FROM email_messages + WHERE user_id = ? + AND updated_at > ? + AND updated_at <= ? + ORDER BY updated_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.watermarkAt, + input.upperBoundAt, + input.limit, + ) + .all() + : await input.db + .prepare( + `SELECT id, updated_at + FROM email_messages + WHERE user_id = ? + AND updated_at <= ? + AND ( + updated_at > ? + OR (updated_at = ? AND id > ?) + ) + ORDER BY updated_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + input.upperBoundAt, + input.cursor.updatedAt, + input.cursor.updatedAt, + input.cursor.id, + input.limit, + ) + .all() + return result.results ?? [] +} + +export async function persistUserParityProgress(input: { + db: D1Database + userId: string + nowIso: string + matchingSince: string | null + mismatchCount: number + contentWatermarkAt: string | null + messageCursor: MailboxParityBackfillCursor | null + messagesCompletedAt: string | null + orphanEventCursor: MailboxParityBackfillCursor | null + orphanEventsCompletedAt: string | null + lastError: string | null +}) { + await input.db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = ?, + mailbox_parity_matching_since = ?, + mailbox_parity_mismatch_count = ?, + mailbox_parity_last_error = ?, + mailbox_parity_content_watermark_at = ?, + mailbox_parity_message_backfill_cursor_created_at = ?, + mailbox_parity_message_backfill_cursor_id = ?, + mailbox_parity_message_backfill_completed_at = ?, + mailbox_parity_orphan_event_backfill_cursor_created_at = ?, + mailbox_parity_orphan_event_backfill_cursor_id = ?, + mailbox_parity_orphan_event_backfill_completed_at = ? + WHERE stable_user_id = ?`, + ) + .bind( + input.nowIso, + input.matchingSince, + input.mismatchCount, + input.lastError, + input.contentWatermarkAt, + input.messageCursor?.createdAt ?? null, + input.messageCursor?.id ?? null, + input.messagesCompletedAt, + input.orphanEventCursor?.createdAt ?? null, + input.orphanEventCursor?.id ?? null, + input.orphanEventsCompletedAt, + input.userId, + ) + .run() +} + +export async function rotateCheckedAt(input: { + db: D1Database + userId: string + nowIso: string + lastError: string | null +}) { + // Clears matching_since so a poison/error tick cannot preserve a false soak. + // Zero-row updates (deleted/deleting race) are harmless. + await input.db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = ?, + mailbox_parity_last_error = ?, + mailbox_parity_matching_since = NULL + WHERE stable_user_id = ?`, + ) + .bind(input.nowIso, input.lastError, input.userId) + .run() +} diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts new file mode 100644 index 0000000000..119318cdef --- /dev/null +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -0,0 +1,528 @@ +import { expect, test, vi } from 'vitest' +import { + getOwnerEmailMessageById, + isMailboxReadCutoverEnabled, + mailboxMessageToEmailMessageRecord, + mailboxReadCutoverCheckedAtMaxAgeMs, + mailboxReadCutoverFlagKey, + mailboxReadCutoverSoakMs, +} from './mailbox-read-cutover.ts' +import { type MailboxMessageRecord } from './mailbox-types.ts' +import { type EmailMessageRecord } from './types.ts' + +const { getEmailMessageByIdMock } = vi.hoisted(() => ({ + getEmailMessageByIdMock: vi.fn(), +})) + +vi.mock('./repo.ts', () => ({ + getEmailMessageById: (...args: Array) => + getEmailMessageByIdMock(...args), +})) + +const nowIso = '2026-08-01T12:00:00.000Z' +const nowMs = Date.parse(nowIso) + +function hoursAgoIso(hours: number) { + return new Date(nowMs - hours * 60 * 60 * 1000).toISOString() +} + +function hoursFromNowIso(hours: number) { + return new Date(nowMs + hours * 60 * 60 * 1000).toISOString() +} + +type ParityRow = { + stable_user_id: string + mailbox_parity_matching_since: string | null + mailbox_parity_checked_at: string | null + mailbox_parity_mismatch_count: number +} + +type FlagState = { + overrideEnabled: boolean | null + globalEnabled: boolean | null +} + +function createCutoverTestDb(input: { + parityByUserId?: Map + flags?: FlagState +}) { + const parityByUserId = input.parityByUserId ?? new Map() + const flags: FlagState = input.flags ?? { + overrideEnabled: null, + globalEnabled: null, + } + + function normalize(query: string) { + return query.replace(/\s+/g, ' ').trim().toLowerCase() + } + + function createStatement(query: string, params: Array = []) { + const normalized = normalize(query) + return { + bind(...nextParams: Array) { + return createStatement(query, nextParams) + }, + async first() { + if ( + normalized.includes('from feature_flag_user_overrides') && + normalized.includes('where flag_key = ? and user_id = ?') + ) { + expect(params[0]).toBe(mailboxReadCutoverFlagKey) + if (flags.overrideEnabled === null) return null + return { enabled: flags.overrideEnabled ? 1 : 0 } as T + } + if ( + normalized.includes('from feature_flags') && + normalized.includes('where key = ?') + ) { + expect(params[0]).toBe(mailboxReadCutoverFlagKey) + if (flags.globalEnabled === null) return null + return { + enabled: flags.globalEnabled ? 1 : 0, + rollout_percent: null, + } as T + } + if ( + normalized.includes('from users') && + normalized.includes('mailbox_parity_matching_since') + ) { + const userId = Number(params[0]) + const row = parityByUserId.get(userId) + if (row === 'error') throw new Error('d1 parity read failed') + if (row === undefined || row === null) return null + return row as T + } + throw new Error(`Unsupported first query: ${query}`) + }, + } + } + + return { + prepare(query: string) { + return createStatement(query) + }, + } as unknown as D1Database +} + +function baseMailboxMessage( + overrides?: Partial, +): MailboxMessageRecord { + return { + id: 'msg-1', + direction: 'inbound', + inboxId: 'inbox-1', + threadId: 'thread-1', + senderIdentityId: null, + fromAddress: 'sender@example.com', + envelopeFrom: null, + toAddresses: ['owner@example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: 'Hello', + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: { 'x-test': '1' }, + authResults: null, + textBody: 'body', + htmlBody: null, + rawMimeKey: 'user-aaa/msg-1.eml', + rawSize: 32, + processingStatus: 'stored', + classification: 'accepted', + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: '2026-07-01T12:00:00.000Z', + sentAt: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + ...overrides, + } +} + +function baseEmailMessage( + overrides?: Partial, +): EmailMessageRecord { + return { + ...mailboxMessageToEmailMessageRecord(baseMailboxMessage(), 'user-aaa'), + ...overrides, + } +} + +function soakedParity(stableUserId = 'user-aaa'): ParityRow { + return { + stable_user_id: stableUserId, + mailbox_parity_matching_since: hoursAgoIso(30), + mailbox_parity_checked_at: hoursAgoIso(1), + mailbox_parity_mismatch_count: 0, + } +} + +function fakeMailboxEnv(getMessage: ReturnType) { + const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) + const get = vi.fn(() => ({ getMessage })) + return { + env: { + MAILBOX: { + idFromName, + get, + } as unknown as DurableObjectNamespace, + }, + idFromName, + get, + } +} + +test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch parity', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const d1Message = baseEmailMessage() + getEmailMessageByIdMock.mockResolvedValue(d1Message) + + // Registry default: flag off → D1, even with soaked parity. + const defaultOffDb = createCutoverTestDb({ + flags: { overrideEnabled: null, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: defaultOffDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + const getMessage = vi.fn(async () => baseMailboxMessage()) + const { env: mailboxEnv, idFromName } = fakeMailboxEnv(getMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...mailboxEnv, APP_DB: defaultOffDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getEmailMessageByIdMock).toHaveBeenCalledWith({ + db: defaultOffDb, + userId: stableUserId, + messageId: 'msg-1', + }) + expect(getMessage).not.toHaveBeenCalled() + expect(idFromName).not.toHaveBeenCalled() + + // Override/global on but no parity → still D1. + for (const flags of [ + { overrideEnabled: true as boolean | null, globalEnabled: null }, + { overrideEnabled: null, globalEnabled: true as boolean | null }, + ]) { + getEmailMessageByIdMock.mockClear() + const noParityDb = createCutoverTestDb({ + flags, + parityByUserId: new Map([[dbUserId, null]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: noParityDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + await expect( + getOwnerEmailMessageById({ + env: { ...mailboxEnv, APP_DB: noParityDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getEmailMessageByIdMock).toHaveBeenCalled() + } + + // Young matching_since (< 24h soak). + const youngDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_matching_since: hoursAgoIso(23), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: youngDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Stale checked_at (> 2h). + const staleDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_checked_at: hoursAgoIso(3), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: staleDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Mismatch count blocks cutover. + const mismatchedDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_mismatch_count: 1, + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: mismatchedDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Cross-user parity row denied (stable_user_id must match exactly). + const crossUserDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity('user-other')]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: crossUserDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + // Soaked + fresh + flag on → Mailbox DO; maps userId from the caller. + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: readyDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(true) + + getEmailMessageByIdMock.mockClear() + const doMessage = baseMailboxMessage({ subject: 'From DO' }) + const doGetMessage = vi.fn(async () => doMessage) + const doEnv = fakeMailboxEnv(doGetMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...doEnv.env, APP_DB: readyDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual( + mailboxMessageToEmailMessageRecord(doMessage, stableUserId), + ) + expect(doEnv.idFromName).toHaveBeenCalledWith(stableUserId) + expect(doGetMessage).toHaveBeenCalledWith({ messageId: 'msg-1' }) + expect(getEmailMessageByIdMock).not.toHaveBeenCalled() + + // Boundary: matching_since exactly soak age and checked_at exactly max age. + const boundaryDb = createCutoverTestDb({ + flags: { overrideEnabled: null, globalEnabled: true }, + parityByUserId: new Map([ + [ + dbUserId, + { + stable_user_id: stableUserId, + mailbox_parity_matching_since: new Date( + nowMs - mailboxReadCutoverSoakMs, + ).toISOString(), + mailbox_parity_checked_at: new Date( + nowMs - mailboxReadCutoverCheckedAtMaxAgeMs, + ).toISOString(), + mailbox_parity_mismatch_count: 0, + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: boundaryDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(true) +}) + +test('mailbox-read-cutover fails closed on invalid now and future parity timestamps', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + + await expect( + isMailboxReadCutoverEnabled({ + db: readyDb, + dbUserId, + stableUserId, + now: 'not-a-timestamp', + }), + ).resolves.toBe(false) + await expect( + isMailboxReadCutoverEnabled({ + db: readyDb, + dbUserId, + stableUserId, + now: new Date(Number.NaN), + }), + ).resolves.toBe(false) + + const futureCheckedDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_checked_at: hoursFromNowIso(1), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: futureCheckedDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + const futureMatchingDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_matching_since: hoursFromNowIso(1), + mailbox_parity_checked_at: hoursAgoIso(1), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: futureMatchingDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) +}) + +test('mailbox-read-cutover fails closed on D1 parity errors and does not fall back from DO', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const d1Message = baseEmailMessage() + getEmailMessageByIdMock.mockResolvedValue(d1Message) + + const parityErrorDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, 'error']]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: parityErrorDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + const getMessage = vi.fn(async () => baseMailboxMessage()) + const { env } = fakeMailboxEnv(getMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...env, APP_DB: parityErrorDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getMessage).not.toHaveBeenCalled() + expect(getEmailMessageByIdMock).toHaveBeenCalled() + + // When cutover is on, DO errors propagate — no silent D1 fallback. + getEmailMessageByIdMock.mockClear() + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + const failingGet = vi.fn(async () => { + throw new Error('mailbox unavailable') + }) + const failingEnv = fakeMailboxEnv(failingGet) + await expect( + getOwnerEmailMessageById({ + env: { ...failingEnv.env, APP_DB: readyDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).rejects.toThrow(/mailbox unavailable/) + expect(failingGet).toHaveBeenCalled() + expect(getEmailMessageByIdMock).not.toHaveBeenCalled() +}) + +test('mailboxMessageToEmailMessageRecord maps DO row with supplied stable userId', () => { + const mapped = mailboxMessageToEmailMessageRecord( + baseMailboxMessage({ id: 'msg-map' }), + 'user-xyz', + ) + expect(mapped).toMatchObject({ + id: 'msg-map', + userId: 'user-xyz', + direction: 'inbound', + subject: 'Hello', + rawMimeKey: 'user-aaa/msg-1.eml', + }) + expect(mapped).not.toHaveProperty('ownerId') +}) diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts new file mode 100644 index 0000000000..fa3501ebce --- /dev/null +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -0,0 +1,195 @@ +/** + * Default-off, parity-gated Mailbox read cutover (prepared; not wired). + * + * Gate requires both: + * 1. feature flag `mailbox-read-cutover` enabled for the numeric `users.id` + * 2. per-user parity soak on `users`: `mailbox_parity_matching_since` at least + * {@link mailboxReadCutoverSoakMs} old, `mailbox_parity_checked_at` fresh + * within {@link mailboxReadCutoverCheckedAtMaxAgeMs}, + * `mailbox_parity_mismatch_count === 0`, and `stable_user_id` exact match + * + * D1 errors, missing/incomplete parity state, and an off flag all fail closed + * to D1-authoritative reads. When the gate is on, Mailbox DO errors propagate + * with no D1 fallback (fallback would hide parity bugs). + * + * Exposure recording is intentionally omitted here: no live caller uses this + * adapter yet. Future read-path wiring owns the exposure chokepoint (the + * existing session/MCP evaluation caches once a live gate evaluates the flag, + * or an explicit `recordFeatureFlagExposures` at the first switched call site). + */ + +import { isFeatureEnabled } from '#worker/feature-flags/service.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { type MailboxMessageRecord } from './mailbox-types.ts' +import { getEmailMessageById } from './repo.ts' +import { type EmailMessageRecord } from './types.ts' + +/** Continuous exact parity must hold at least this long before cutover. */ +export const mailboxReadCutoverSoakMs = 24 * 60 * 60 * 1000 + +/** Parity `checked_at` must be newer than this (hourly lane + slack). */ +export const mailboxReadCutoverCheckedAtMaxAgeMs = 2 * 60 * 60 * 1000 + +export const mailboxReadCutoverFlagKey = 'mailbox-read-cutover' as const + +type MailboxParityStateRow = { + stable_user_id: string + mailbox_parity_matching_since: string | null + mailbox_parity_checked_at: string | null + mailbox_parity_mismatch_count: number +} + +export type MailboxReadCutoverEnv = MailboxEnv & { + APP_DB: D1Database +} + +/** + * Resolve an optional clock override. `undefined` uses wall clock; invalid + * Date/string values return null so the gate fails closed (never Date.now()). + */ +function resolveNowMs(now?: Date | string): number | null { + if (now === undefined) return Date.now() + if (now instanceof Date) { + const ms = now.getTime() + return Number.isFinite(ms) ? ms : null + } + const parsed = Date.parse(now) + return Number.isFinite(parsed) ? parsed : null +} + +/** + * Fail-closed cutover eligibility for one account. + * Never throws: D1 / flag evaluation failures return false. + */ +export async function isMailboxReadCutoverEnabled(input: { + db: D1Database + dbUserId: number + stableUserId: string + now?: Date | string +}): Promise { + try { + const flagEnabled = await isFeatureEnabled( + input.db, + mailboxReadCutoverFlagKey, + input.dbUserId, + ) + if (!flagEnabled) return false + + const row = await input.db + .prepare( + `SELECT stable_user_id, + mailbox_parity_matching_since, + mailbox_parity_checked_at, + mailbox_parity_mismatch_count + FROM users + WHERE id = ?`, + ) + .bind(input.dbUserId) + .first() + + if (!row) return false + if (row.stable_user_id !== input.stableUserId) return false + if (row.mailbox_parity_mismatch_count !== 0) return false + if ( + row.mailbox_parity_matching_since == null || + row.mailbox_parity_checked_at == null + ) { + return false + } + + const matchingSinceMs = Date.parse(row.mailbox_parity_matching_since) + const checkedAtMs = Date.parse(row.mailbox_parity_checked_at) + if (!Number.isFinite(matchingSinceMs) || !Number.isFinite(checkedAtMs)) { + return false + } + + const nowMs = resolveNowMs(input.now) + if (nowMs === null) return false + // Future timestamps are not trustworthy (clock skew / bad writes). + if (matchingSinceMs > nowMs) return false + if (checkedAtMs > nowMs) return false + if (nowMs - matchingSinceMs < mailboxReadCutoverSoakMs) return false + if (nowMs - checkedAtMs > mailboxReadCutoverCheckedAtMaxAgeMs) return false + + return true + } catch { + return false + } +} + +/** Map a Mailbox DO message row back to the D1 `EmailMessageRecord` shape. */ +export function mailboxMessageToEmailMessageRecord( + message: MailboxMessageRecord, + userId: string, +): EmailMessageRecord { + return { + id: message.id, + direction: message.direction, + userId, + inboxId: message.inboxId, + threadId: message.threadId, + senderIdentityId: message.senderIdentityId, + fromAddress: message.fromAddress, + envelopeFrom: message.envelopeFrom, + toAddresses: message.toAddresses, + ccAddresses: message.ccAddresses, + bccAddresses: message.bccAddresses, + replyToAddresses: message.replyToAddresses, + subject: message.subject, + messageIdHeader: message.messageIdHeader, + inReplyToHeader: message.inReplyToHeader, + references: message.references, + headers: message.headers, + authResults: message.authResults, + textBody: message.textBody, + htmlBody: message.htmlBody, + rawMimeKey: message.rawMimeKey, + rawSize: message.rawSize, + processingStatus: message.processingStatus, + classification: message.classification, + classificationReason: message.classificationReason, + providerMessageId: message.providerMessageId, + deliveryStatus: message.deliveryStatus, + deliveryStatusAt: message.deliveryStatusAt, + error: message.error, + receivedAt: message.receivedAt, + sentAt: message.sentAt, + createdAt: message.createdAt, + updatedAt: message.updatedAt, + } +} + +/** + * Prepared owner-scoped message get. Selects D1 or Mailbox via the cutover + * gate. Not wired into existing readers in this PR. + */ +export async function getOwnerEmailMessageById(input: { + env: MailboxReadCutoverEnv + dbUserId: number + stableUserId: string + messageId: string + now?: Date | string +}): Promise { + const useMailbox = await isMailboxReadCutoverEnabled({ + db: input.env.APP_DB, + dbUserId: input.dbUserId, + stableUserId: input.stableUserId, + now: input.now, + }) + + if (!useMailbox) { + return getEmailMessageById({ + db: input.env.APP_DB, + userId: input.stableUserId, + messageId: input.messageId, + }) + } + + const message = await mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }).getMessage({ messageId: input.messageId }) + + if (!message) return null + return mailboxMessageToEmailMessageRecord(message, input.stableUserId) +} diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts new file mode 100644 index 0000000000..37416204b4 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile.node.test.ts @@ -0,0 +1,783 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import type * as MailboxMirrorModule from './mailbox-mirror.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const mocks = vi.hoisted(() => ({ + mirrorMailboxMessageGraphFromD1: vi.fn(), + mirrorMailboxDeliveryEventFromD1: vi.fn(), + awaitMailboxMirrorRpc: vi.fn(), + recordMailboxParityEvent: vi.fn(), + mailboxRpc: vi.fn(), +})) + +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, + mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, +})) + +vi.mock('./mailbox-mirror.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + awaitMailboxMirrorRpc: mocks.awaitMailboxMirrorRpc, + } +}) + +vi.mock('./mailbox-parity-events.ts', () => ({ + recordMailboxParityEvent: mocks.recordMailboxParityEvent, +})) + +vi.mock('./mailbox-client.ts', () => ({ + mailboxRpc: mocks.mailboxRpc, +})) + +const { + countD1MailboxParity, + listOrphanDeliveryEventBackfillPage, + listUsersForMailboxParity, + mailboxParityMessagePageSize, + reconcileMailboxParity, +} = await import('./mailbox-reconcile.ts') + +async function createParityDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureUsersTestSchema({ db }) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +async function insertUser(input: { + db: D1Database + userId: string + email?: string + deletingAt?: string | null + checkedAt?: string | null +}) { + await input.db + .prepare( + `INSERT INTO users ( + username, email, password_hash, stable_user_id, deleting_at, + mailbox_parity_checked_at + ) VALUES (?, ?, 'hash', ?, ?, ?)`, + ) + .bind( + `u-${crypto.randomUUID().slice(0, 8)}`, + input.email ?? `${crypto.randomUUID()}@example.test`, + input.userId, + input.deletingAt ?? null, + input.checkedAt ?? null, + ) + .run() +} + +async function insertMessage(input: { + db: D1Database + userId: string + id: string + createdAt: string +}) { + await input.db + .prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, text_body, raw_size, + processing_status, created_at, updated_at + ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, + 'stored', ?, ?)`, + ) + .bind(input.id, input.userId, input.createdAt, input.createdAt) + .run() +} + +async function insertOrphanEvent(input: { + db: D1Database + userId: string + id: string + createdAt: string +}) { + await input.db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind(input.id, input.userId, input.createdAt) + .run() +} + +async function parkUser(db: D1Database, userId: string) { + await db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-01-01T00:00:00.000Z' + WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() +} + +async function readParityRow(db: D1Database, userId: string) { + return db + .prepare( + `SELECT + mailbox_parity_checked_at AS checkedAt, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_last_error AS lastError, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, + mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, + mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + FROM users + WHERE stable_user_id = ?`, + ) + .bind(userId) + .first<{ + checkedAt: string | null + matchingSince: string | null + mismatchCount: number + lastError: string | null + contentWatermarkAt: string | null + messageCursorCreatedAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + orphanCursorCreatedAt: string | null + orphanCursorId: string | null + orphanEventsCompletedAt: string | null + }>() +} + +function successGraph(messageId: string) { + return { + messageId, + message: { status: 'mirrored' as const }, + events: [], + eventsTruncated: false, + } +} + +function mockDoCounts(counts: { + threads: number + messages: number + attachments: number + deliveryEvents: number +}) { + mocks.awaitMailboxMirrorRpc.mockResolvedValue({ + ok: true, + value: counts, + }) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => counts), + })) +} + +test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const ownerA = 'a'.repeat(64) + const ownerB = 'b'.repeat(64) + const deleting = 'c'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + + await insertUser({ db, userId: ownerA, checkedAt: null }) + await insertUser({ + db, + userId: ownerB, + checkedAt: '2026-07-01T00:00:00.000Z', + }) + await insertUser({ + db, + userId: deleting, + deletingAt: '2026-07-31T00:00:00.000Z', + checkedAt: null, + }) + + const messageIds = Array.from( + { length: mailboxParityMessagePageSize + 1 }, + (_, index) => `msg-${String(index + 1).padStart(2, '0')}`, + ) + for (const [index, id] of messageIds.entries()) { + await insertMessage({ + db, + userId: ownerA, + id, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + await insertOrphanEvent({ + db, + userId: ownerA, + id: 'orphan-1', + createdAt: '2026-07-02T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: ownerB, + id: 'b-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: systemEmailOwnerId, + id: 'sys-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertOrphanEvent({ + db, + userId: deleting, + id: 'del-orphan', + createdAt: '2026-07-01T00:00:00.000Z', + }) + + // NULL checked_at sorts first under ORDER BY checked_at ASC (index-friendly). + const discovered = await listUsersForMailboxParity({ db, limit: 10 }) + expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) + expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) + expect(discovered.map((row) => row.userId)).not.toContain(deleting) + + const orphanPage = await listOrphanDeliveryEventBackfillPage({ + db, + userId: ownerA, + cursor: null, + limit: 10, + }) + expect(orphanPage).toEqual([ + { id: 'orphan-1', created_at: '2026-07-02T00:00:00.000Z' }, + ]) + + await db + .prepare( + `INSERT INTO email_threads ( + id, user_id, subject_normalized, last_message_at, created_at, updated_at + ) VALUES ('thread-a', ?, 'hi', ?, ?, ?)`, + ) + .bind( + ownerA, + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + ) + .run() + await db + .prepare( + `INSERT INTO email_attachments ( + id, message_id, content_type, size, storage_kind, created_at + ) VALUES ('att-1', ?, 'text/plain', 1, 'unavailable', ?)`, + ) + .bind(messageIds[0]!, '2026-07-01T00:00:00.000Z') + .run() + + await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ + threads: 1, + messages: mailboxParityMessagePageSize + 1, + attachments: 1, + deliveryEvents: 1, + }) + + let graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + graphCalls += 1 + if (graphCalls === mailboxParityMessagePageSize) { + return { + messageId: input.messageId, + message: { status: 'timeout' as const }, + events: [], + eventsTruncated: false, + } + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + mocks.recordMailboxParityEvent.mockClear() + + const env = { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } + + // Seed a false soak that must clear on mirror timeout / backfill work. + await db + .prepare( + `UPDATE users + SET mailbox_parity_matching_since = '2026-07-01T00:00:00.000Z', + mailbox_parity_mismatch_count = 2 + WHERE stable_user_id = ?`, + ) + .bind(ownerA) + .run() + + await parkUser(db, ownerB) + const first = await reconcileMailboxParity({ + env, + now, + batchSize: 1, + }) + expect(first).toEqual({ + scanned: 1, + backfilled: mailboxParityMessagePageSize - 1, + compared: 0, + matched: 0, + mismatched: 0, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-message-backfill-retryable', + ownerA, + 'error-or-timeout', + ) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.every( + (call) => call[0].userId === ownerA, + ), + ).toBe(true) + expect(mocks.mailboxRpc).not.toHaveBeenCalled() + + const afterFirst = await readParityRow(db, ownerA) + expect(afterFirst).toMatchObject({ + checkedAt: now.toISOString(), + messageCursorId: messageIds[mailboxParityMessagePageSize - 2], + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + // Baseline is persisted at first backfill attempt start and retained. + contentWatermarkAt: now.toISOString(), + matchingSince: null, + mismatchCount: 2, + lastError: 'message backfill mirror error or timeout', + }) + + // Finish backfill; compare mismatches → reopen completed_at, retain cursors, + // increment consecutive mismatch count, keep matching_since cleared. + graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mocks.recordMailboxParityEvent.mockClear() + await parkUser(db, ownerB) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + const mismatch = await reconcileMailboxParity({ + env, + now: mismatchNow, + batchSize: 1, + }) + expect(mismatch).toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'orphan-1' }), + ) + expect(mocks.recordMailboxParityEvent).toHaveBeenCalledTimes(4) + + const afterMismatch = await readParityRow(db, ownerA) + expect(afterMismatch).toMatchObject({ + checkedAt: mismatchNow.toISOString(), + matchingSince: null, + mismatchCount: 3, + contentWatermarkAt: mismatchNow.toISOString(), + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + messageCursorId: messageIds[mailboxParityMessagePageSize], + orphanCursorId: 'orphan-1', + lastError: null, + }) + + // Live new row after retained cursor must be repaired on the reopen pass. + await insertMessage({ + db, + userId: ownerA, + id: 'msg-live-new', + createdAt: '2026-07-03T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mockDoCounts({ + threads: 1, + messages: mailboxParityMessagePageSize + 2, + attachments: 1, + deliveryEvents: 1, + }) + await parkUser(db, ownerB) + const rematchNow = new Date('2026-08-01T12:00:00.000Z') + const rematch = await reconcileMailboxParity({ + env, + now: rematchNow, + batchSize: 1, + }) + expect(rematch).toEqual({ + scanned: 1, + backfilled: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, messageId: 'msg-live-new' }), + ) + const afterRematch = await readParityRow(db, ownerA) + expect(afterRematch).toMatchObject({ + matchingSince: rematchNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: rematchNow.toISOString(), + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + messageCursorId: 'msg-live-new', + }) + + // 24h soak preservation: exact completed compare with empty content window + // advances the watermark and keeps matching_since. + await parkUser(db, ownerB) + const soakNow = new Date('2026-08-02T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterSoak = await readParityRow(db, ownerA) + expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) + expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) + expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) + expect(afterSoak?.mismatchCount).toBe(0) + + // Compare-path error rotates checked_at and clears matching_since. + consoleWarn.mockClear() + mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( + new Error('count boom\nwith\tcontrol'), + ) + await parkUser(db, ownerB) + const errorNow = new Date('2026-08-02T13:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: errorNow, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterError = await readParityRow(db, ownerA) + expect(afterError?.checkedAt).toBe(errorNow.toISOString()) + expect(afterError?.lastError).toBe('count boom with control') + expect(afterError?.matchingSince).toBeNull() + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-reconcile-user-failed', + ownerA, + expect.any(Error), + ) + + // Discovery stays D1-only: mailboxRpc is never used to enumerate candidates. + const rpcCallsBefore = mocks.mailboxRpc.mock.calls.length + const discoveryOnly = await listUsersForMailboxParity({ db, limit: 5 }) + expect(discoveryOnly.length).toBeGreaterThan(0) + expect(mocks.mailboxRpc).toHaveBeenCalledTimes(rpcCallsBefore) +}) + +test('mailbox parity content watermark repairs equal-count updates and soak', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'd'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-a', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-b', + createdAt: '2026-07-01T00:00:01.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + + const env = { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } + + // Baseline at first-attempt start; empty same-run window; then compare. + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 2, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterBaseline = await readParityRow(db, owner) + expect(afterBaseline).toMatchObject({ + matchingSince: now.toISOString(), + contentWatermarkAt: now.toISOString(), + mismatchCount: 0, + }) + + // Classification-like update: same counts, newer updated_at past watermark. + const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-a'`, + ) + .bind(sharedUpdatedAt) + .run() + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-b'`, + ) + .bind(sharedUpdatedAt) + .run() + + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + const repairNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(repairNow) + await expect( + reconcileMailboxParity({ env, now: repairNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 2, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const replayedIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ) + expect(replayedIds).toEqual(['msg-a', 'msg-b']) + const afterRepair = await readParityRow(db, owner) + // Successful content sweep + exact compare keeps the soak window. + expect(afterRepair?.matchingSince).toBe(now.toISOString()) + expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) + expect(afterRepair?.mismatchCount).toBe(0) + + // Equal-timestamp rows after the advanced watermark are both replayed. + const equalStamp = '2026-08-01T11:15:00.000Z' + await db + .prepare( + `UPDATE email_messages SET updated_at = ? WHERE id IN ('msg-a', 'msg-b')`, + ) + .bind(equalStamp) + .run() + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + const equalNow = new Date('2026-08-01T12:00:00.000Z') + vi.setSystemTime(equalNow) + await expect( + reconcileMailboxParity({ env, now: equalNow, batchSize: 1 }), + ).resolves.toMatchObject({ + backfilled: 2, + compared: 1, + matched: 1, + failed: 0, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-a', 'msg-b']) + expect((await readParityRow(db, owner))?.matchingSince).toBe( + now.toISOString(), + ) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + equalNow.toISOString(), + ) + + // Failure mid-window: watermark does not advance; matching_since clears. + await db + .prepare( + `UPDATE email_messages + SET updated_at = '2026-08-01T12:30:00.000Z' + WHERE id IN ('msg-a', 'msg-b')`, + ) + .run() + await db + .prepare( + `UPDATE users + SET mailbox_parity_matching_since = ? + WHERE stable_user_id = ?`, + ) + .bind(now.toISOString(), owner) + .run() + let contentCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + contentCalls += 1 + if (contentCalls === 1) { + return { + messageId: input.messageId, + message: { status: 'timeout' as const }, + events: [], + eventsTruncated: false, + } + } + return successGraph(input.messageId) + }, + ) + consoleWarn.mockClear() + const failNow = new Date('2026-08-01T13:00:00.000Z') + vi.setSystemTime(failNow) + await expect( + reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-content-replay-retryable', + owner, + 'error-or-timeout', + ) + const afterFail = await readParityRow(db, owner) + expect(afterFail).toMatchObject({ + contentWatermarkAt: equalNow.toISOString(), + matchingSince: null, + checkedAt: failNow.toISOString(), + lastError: 'content replay mirror error or timeout', + mismatchCount: 0, + }) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity replays message updated during initial backfill', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'e'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + const midUpdateAt = new Date('2026-08-01T10:00:05.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-early', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-late', + createdAt: '2026-07-01T00:00:01.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + if (input.messageId === 'msg-early') { + // Concurrent classification-like update after the graph mirror. + vi.setSystemTime(midUpdateAt) + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-early'`, + ) + .bind(midUpdateAt.toISOString()) + .run() + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + + const env = { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } + + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + + const mirroredIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ) + expect(mirroredIds).toEqual(['msg-early', 'msg-late', 'msg-early']) + const after = await readParityRow(db, owner) + expect(after).toMatchObject({ + matchingSince: now.toISOString(), + contentWatermarkAt: midUpdateAt.toISOString(), + mismatchCount: 0, + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + }) + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts new file mode 100644 index 0000000000..a8d3dcc097 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -0,0 +1,787 @@ +import { truncateToUtf8Bytes } from '@kody-internal/shared/backup-restore-safety.ts' +import { mailboxRpc } from './mailbox-client.ts' +import { + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, + type MailboxLiveMirrorGraphSummary, +} from './mailbox-live-mirror.ts' +import { + awaitMailboxMirrorRpc, + mailboxMirrorRpcTimeoutMs, + type MailboxMirrorResult, +} from './mailbox-mirror.ts' +import { + recordMailboxParityEvent, + type MailboxParityOperation, +} from './mailbox-parity-events.ts' +import { + countD1MailboxParity, + listContentReplayPage, + listMessageBackfillPage, + listOrphanDeliveryEventBackfillPage, + listUsersForMailboxParity, + loadUserParityState, + persistUserParityProgress, + rotateCheckedAt, + type MailboxParityContentCursor, + type MailboxParityUserState, +} from './mailbox-parity-repo.ts' +import { type MailboxCountResult } from './mailbox-types.ts' + +/** + * Bounded hourly Mailbox backfill + count-parity reconciler. + * + * Discovers a small oldest-first page of non-deleting D1 mail owners (no DO + * enumeration), mirrors message graphs then message_id-null orphan delivery + * events through existing live-mirror helpers, then keyset-replays messages + * updated after the content watermark before comparing owner-scoped D1 counts + * with `countMailbox`. D1 remains mail authority; this lane never mutates + * email_* rows and does not flip read authority. + */ + +export const mailboxParityUserBatchSize = 8 +export const mailboxParityMessagePageSize = 10 +export const mailboxParityOrphanEventPageSize = 25 +export const mailboxParityContentPageSize = 10 +export const mailboxParityTimeBudgetMs = 10_000 +export const mailboxParityLastErrorMaxBytes = 512 + +export type MailboxParityReconcileEnv = MailboxLiveMirrorEnv & { + APP_DB: D1Database +} + +export type MailboxParityReconcileMetrics = { + scanned: number + backfilled: number + compared: number + matched: number + mismatched: number + failed: number +} + +export { + countD1MailboxParity, + listOrphanDeliveryEventBackfillPage, + listUsersForMailboxParity, +} + +function isRetryableMirrorResult(result: MailboxMirrorResult): boolean { + return result.status === 'error' || result.status === 'timeout' +} + +function graphMirrorSucceeded(summary: MailboxLiveMirrorGraphSummary): boolean { + if (isRetryableMirrorResult(summary.message)) return false + return !summary.events.some((event) => isRetryableMirrorResult(event.result)) +} + +function safeParityErrorText(error: unknown): string { + const raw = error instanceof Error ? error.message : String(error) + let cleaned = '' + for (const character of raw) { + const code = character.codePointAt(0) ?? 0 + cleaned += code < 0x20 || code === 0x7f ? ' ' : character + } + cleaned = cleaned.trim() + return truncateToUtf8Bytes( + cleaned.length > 0 ? cleaned : 'unknown error', + mailboxParityLastErrorMaxBytes, + ) +} + +/** + * Load one orphan delivery-event projection from D1 and mirror it. + * Thin cohesive wrapper over {@link mirrorMailboxDeliveryEventFromD1}. + */ +export async function mirrorMailboxOrphanDeliveryEventFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + eventId: string +}): Promise { + return mirrorMailboxDeliveryEventFromD1(input) +} + +function emitCountComparisons(input: { + env: MailboxParityReconcileEnv + userId: string + d1: MailboxCountResult + mailbox: MailboxCountResult + timestamp: string +}): { matched: boolean } { + const comparisons: Array<{ + operation: MailboxParityOperation + d1Count: number + doCount: number + }> = [ + { + operation: 'compare_threads', + d1Count: input.d1.threads, + doCount: input.mailbox.threads, + }, + { + operation: 'compare_messages', + d1Count: input.d1.messages, + doCount: input.mailbox.messages, + }, + { + operation: 'compare_attachments', + d1Count: input.d1.attachments, + doCount: input.mailbox.attachments, + }, + { + operation: 'compare_delivery_events', + d1Count: input.d1.deliveryEvents, + doCount: input.mailbox.deliveryEvents, + }, + ] + let matched = true + for (const comparison of comparisons) { + const outcome = + comparison.d1Count === comparison.doCount ? 'match' : 'mismatch' + if (outcome === 'mismatch') matched = false + recordMailboxParityEvent(input.env, { + userId: input.userId, + category: 'parity', + operation: comparison.operation, + outcome, + d1Count: comparison.d1Count, + doCount: comparison.doCount, + timestamp: input.timestamp, + }) + } + return { matched } +} + +async function backfillMessagesForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + deadlineMs: number +}): Promise<{ + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean +}> { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listMessageBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.messageCursor, + limit: mailboxParityMessagePageSize, + }) + if (page.length === 0) { + state = { + ...state, + messagesCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + let cursor = state.messageCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (!graphMirrorSucceeded(summary)) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, messageCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } +} + +async function backfillOrphanEventsForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + deadlineMs: number +}): Promise<{ + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean +}> { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt == null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + if (state.orphanEventsCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listOrphanDeliveryEventBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.orphanEventCursor, + limit: mailboxParityOrphanEventPageSize, + }) + if (page.length === 0) { + state = { + ...state, + orphanEventsCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + } + } + + let cursor = state.orphanEventCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, orphanEventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } + } + const result = await mirrorMailboxOrphanDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + eventId: row.id, + }) + if (isRetryableMirrorResult(result)) { + state = { ...state, orphanEventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, orphanEventCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + } +} + +/** + * Baseline is established before the first creation scan. When it still equals + * this run's nowIso, sample wall time so same-run updates after the baseline + * are included; otherwise the run's nowIso is the upper bound. + */ +function resolveContentReplayUpperBound(input: { + nowIso: string + watermarkAt: string +}): string { + let upperBoundAt = input.nowIso + if (input.watermarkAt === input.nowIso) { + const wallIso = new Date().toISOString() + if (wallIso > upperBoundAt) upperBoundAt = wallIso + } + if (upperBoundAt < input.watermarkAt) return input.watermarkAt + return upperBoundAt +} + +/** + * Persist the content watermark at the start of the first backfill attempt so + * updates during creation/orphan mirroring cannot fall before a completion-time + * baseline. Retained across incomplete/error ticks. + */ +async function ensureContentWatermarkBaseline(input: { + db: D1Database + state: MailboxParityUserState + nowIso: string +}): Promise { + if (input.state.contentWatermarkAt != null) return input.state + const state = { + ...input.state, + contentWatermarkAt: input.nowIso, + } + await persistUserParityProgress({ + db: input.db, + userId: state.userId, + nowIso: input.nowIso, + matchingSince: state.matchingSince, + mismatchCount: state.mismatchCount, + contentWatermarkAt: state.contentWatermarkAt, + messageCursor: state.messageCursor, + messagesCompletedAt: state.messagesCompletedAt, + orphanEventCursor: state.orphanEventCursor, + orphanEventsCompletedAt: state.orphanEventsCompletedAt, + lastError: null, + }) + return state +} + +/** + * After initial creation/orphan backfill completes: keyset-replay every owner + * message with updated_at in (watermark, current-now]. Watermark advances only + * when the entire window succeeds; in-tick cursor avoids skipping equal + * timestamps. Always replay — never treat a fresh baseline as an empty skip. + */ +async function replayContentUpdatesForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + nowIso: string + deadlineMs: number +}): Promise<{ + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean + windowComplete: boolean +}> { + const state = input.state + const watermarkAt = state.contentWatermarkAt ?? input.nowIso + const upperBoundAt = resolveContentReplayUpperBound({ + nowIso: input.nowIso, + watermarkAt, + }) + let cursor: MailboxParityContentCursor | null = null + let backfilled = 0 + + while (Date.now() < input.deadlineMs) { + const page = await listContentReplayPage({ + db: input.env.APP_DB, + userId: state.userId, + watermarkAt, + upperBoundAt, + cursor, + limit: mailboxParityContentPageSize, + }) + if (page.length === 0) { + return { + state: { ...state, contentWatermarkAt: upperBoundAt }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + windowComplete: true, + } + } + + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + windowComplete: false, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (!graphMirrorSucceeded(summary)) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + windowComplete: false, + } + } + cursor = { updatedAt: row.updated_at, id: row.id } + backfilled += 1 + } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + windowComplete: false, + } +} + +async function compareParityForUser(input: { + env: MailboxParityReconcileEnv + state: MailboxParityUserState + nowIso: string +}): Promise<{ + state: MailboxParityUserState + matched: boolean +}> { + const d1 = await countD1MailboxParity({ + db: input.env.APP_DB, + userId: input.state.userId, + }) + const raced = await awaitMailboxMirrorRpc( + mailboxRpc({ env: input.env, userId: input.state.userId }).countMailbox(), + mailboxMirrorRpcTimeoutMs, + ) + if (!raced.ok) { + throw new Error('Mailbox countMailbox timed out') + } + const mailbox = raced.value + const { matched } = emitCountComparisons({ + env: input.env, + userId: input.state.userId, + d1, + mailbox, + timestamp: input.nowIso, + }) + if (matched) { + return { + state: { + ...input.state, + matchingSince: input.state.matchingSince ?? input.nowIso, + mismatchCount: 0, + }, + matched: true, + } + } + // Reopen bounded creation/orphan backfill after completed cursors so later + // live rows past the retained keyset are mirrored on the next ticks; keep + // the content watermark so classification updates remain covered. + return { + state: { + ...input.state, + matchingSince: null, + mismatchCount: input.state.mismatchCount + 1, + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + }, + matched: false, + } +} + +async function reconcileOneUser(input: { + env: MailboxParityReconcileEnv + userId: string + nowIso: string + deadlineMs: number +}): Promise<{ + backfilled: number + compared: boolean + matched: boolean + mismatched: boolean + failed: boolean + budgetExhausted: boolean +}> { + const loaded = await loadUserParityState({ + db: input.env.APP_DB, + userId: input.userId, + }) + if (!loaded) { + return { + backfilled: 0, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: false, + } + } + + let state = loaded + let backfilled = 0 + let initialBackfilled = 0 + + const persistIncomplete = async (inputPersist: { + lastError: string | null + }) => { + await persistUserParityProgress({ + db: input.env.APP_DB, + userId: state.userId, + nowIso: input.nowIso, + // Soak cannot survive incomplete/error/backfill ticks. + matchingSince: null, + mismatchCount: state.mismatchCount, + contentWatermarkAt: state.contentWatermarkAt, + messageCursor: state.messageCursor, + messagesCompletedAt: state.messagesCompletedAt, + orphanEventCursor: state.orphanEventCursor, + orphanEventsCompletedAt: state.orphanEventsCompletedAt, + lastError: inputPersist.lastError, + }) + } + + // Baseline before scanning so mid-backfill updates stay inside the replay window. + state = await ensureContentWatermarkBaseline({ + db: input.env.APP_DB, + state, + nowIso: input.nowIso, + }) + + const messagePass = await backfillMessagesForUser({ + env: input.env, + state, + deadlineMs: input.deadlineMs, + }) + state = messagePass.state + initialBackfilled += messagePass.backfilled + backfilled += messagePass.backfilled + if (messagePass.retryableFailure) { + await persistIncomplete({ + lastError: 'message backfill mirror error or timeout', + }) + console.warn( + 'mailbox-parity-message-backfill-retryable', + state.userId, + 'error-or-timeout', + ) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: true, + budgetExhausted: false, + } + } + if (messagePass.budgetExhausted) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: true, + } + } + + const orphanPass = await backfillOrphanEventsForUser({ + env: input.env, + state, + deadlineMs: input.deadlineMs, + }) + state = orphanPass.state + initialBackfilled += orphanPass.backfilled + backfilled += orphanPass.backfilled + if (orphanPass.retryableFailure) { + await persistIncomplete({ + lastError: 'orphan-event backfill mirror error or timeout', + }) + console.warn( + 'mailbox-parity-orphan-event-backfill-retryable', + state.userId, + 'error-or-timeout', + ) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: true, + budgetExhausted: false, + } + } + if (orphanPass.budgetExhausted) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: true, + } + } + + if ( + state.messagesCompletedAt == null || + state.orphanEventsCompletedAt == null + ) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: false, + } + } + + // Initial creation/orphan work ends the continuous soak window. Successful + // content replay below may keep matching_since when the final compare matches. + if (initialBackfilled > 0) { + state = { ...state, matchingSince: null } + } + + const contentPass = await replayContentUpdatesForUser({ + env: input.env, + state, + nowIso: input.nowIso, + deadlineMs: input.deadlineMs, + }) + state = contentPass.state + backfilled += contentPass.backfilled + if (contentPass.retryableFailure) { + await persistIncomplete({ + lastError: 'content replay mirror error or timeout', + }) + console.warn( + 'mailbox-parity-content-replay-retryable', + state.userId, + 'error-or-timeout', + ) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: true, + budgetExhausted: false, + } + } + if (!contentPass.windowComplete) { + await persistIncomplete({ lastError: null }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: contentPass.budgetExhausted, + } + } + + const compared = await compareParityForUser({ + env: input.env, + state, + nowIso: input.nowIso, + }) + state = compared.state + await persistUserParityProgress({ + db: input.env.APP_DB, + userId: state.userId, + nowIso: input.nowIso, + matchingSince: state.matchingSince, + mismatchCount: state.mismatchCount, + contentWatermarkAt: state.contentWatermarkAt, + messageCursor: state.messageCursor, + messagesCompletedAt: state.messagesCompletedAt, + orphanEventCursor: state.orphanEventCursor, + orphanEventsCompletedAt: state.orphanEventsCompletedAt, + lastError: null, + }) + return { + backfilled, + compared: true, + matched: compared.matched, + mismatched: !compared.matched, + failed: false, + budgetExhausted: false, + } +} + +export async function reconcileMailboxParity(input: { + env: MailboxParityReconcileEnv + now?: Date + batchSize?: number +}): Promise { + const now = input.now ?? new Date() + const nowIso = now.toISOString() + const deadlineMs = Date.now() + mailboxParityTimeBudgetMs + const batchSize = input.batchSize ?? mailboxParityUserBatchSize + const users = await listUsersForMailboxParity({ + db: input.env.APP_DB, + limit: batchSize, + }) + + const metrics: MailboxParityReconcileMetrics = { + scanned: 0, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + } + + for (const { userId } of users) { + if (Date.now() >= deadlineMs) break + metrics.scanned += 1 + try { + const result = await reconcileOneUser({ + env: input.env, + userId, + nowIso, + deadlineMs, + }) + metrics.backfilled += result.backfilled + if (result.compared) metrics.compared += 1 + if (result.matched) metrics.matched += 1 + if (result.mismatched) metrics.mismatched += 1 + if (result.failed) metrics.failed += 1 + if (result.budgetExhausted) break + } catch (error) { + metrics.failed += 1 + const lastError = safeParityErrorText(error) + await rotateCheckedAt({ + db: input.env.APP_DB, + userId, + nowIso, + lastError, + }).catch(() => undefined) + console.warn('mailbox-parity-reconcile-user-failed', userId, error) + } + } + + return metrics +} diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts new file mode 100644 index 0000000000..243d0b2086 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -0,0 +1,331 @@ +import { env } from 'cloudflare:workers' +import { expect, test, vi } from 'vitest' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { + countD1MailboxParity, + reconcileMailboxParity, +} from './mailbox-reconcile.ts' +import { rpcFor } from './mailbox-test-helpers.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +async function seedParityUser(input: { + email: string + checkedAt?: string | null +}) { + const userId = await createStableUserIdFromEmail(input.email) + await env.APP_DB.prepare( + `INSERT INTO users ( + username, email, password_hash, plan, stable_user_id, + mailbox_parity_checked_at + ) VALUES (?, ?, 'test-password-hash', 'max', ?, ?)`, + ) + .bind( + `parity-${crypto.randomUUID().slice(0, 8)}`, + input.email, + userId, + input.checkedAt ?? null, + ) + .run() + return userId +} + +async function seedMessageGraph(input: { + userId: string + messageId: string + threadId: string + createdAt: string + withAttachment?: boolean +}) { + await env.APP_DB.prepare( + `INSERT INTO email_threads ( + id, user_id, subject_normalized, last_message_at, created_at, updated_at + ) VALUES (?, ?, 'hello', ?, ?, ?)`, + ) + .bind( + input.threadId, + input.userId, + input.createdAt, + input.createdAt, + input.createdAt, + ) + .run() + await env.APP_DB.prepare( + `INSERT INTO email_messages ( + id, direction, user_id, thread_id, from_address, subject, text_body, + raw_size, processing_status, created_at, updated_at + ) VALUES (?, 'outbound', ?, ?, 'from@example.test', 'Hello', 'body', 4, + 'sent', ?, ?)`, + ) + .bind( + input.messageId, + input.userId, + input.threadId, + input.createdAt, + input.createdAt, + ) + .run() + if (input.withAttachment) { + await env.APP_DB.prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, created_at + ) VALUES (?, ?, 'note.txt', 'text/plain', 4, 'unavailable', ?)`, + ) + .bind(`att-${input.messageId}`, input.messageId, input.createdAt) + .run() + } + await env.APP_DB.prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, ?, ?, 'sent', 'kody', '{}', ?)`, + ) + .bind( + `evt-${input.messageId}`, + input.messageId, + input.userId, + input.createdAt, + ) + .run() +} + +async function seedOrphanEvent(input: { + userId: string + eventId: string + createdAt: string +}) { + await env.APP_DB.prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind(input.eventId, input.userId, input.createdAt) + .run() +} + +async function readParityState(userId: string) { + return env.APP_DB.prepare( + `SELECT + mailbox_parity_checked_at AS checkedAt, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, + mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt, + mailbox_parity_last_error AS lastError + FROM users + WHERE stable_user_id = ?`, + ) + .bind(userId) + .first<{ + checkedAt: string | null + matchingSince: string | null + mismatchCount: number + contentWatermarkAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + orphanCursorId: string | null + orphanEventsCompletedAt: string | null + lastError: string | null + }>() +} + +async function parkOther(userId: string) { + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z' + WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() +} + +test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves soak', async () => { + silenceIncidentalRuntimeWarnings() + // Keep wall time aligned with each tick's nowIso so same-run content upper + // bounds stay deterministic (without freezing RPC timeout clocks). + vi.useFakeTimers({ shouldAdvanceTime: true }) + await ensureUsersTestSchema({ db: env.APP_DB }) + await ensureEmailTestSchema(env.APP_DB) + + try { + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z'`, + ).run() + + const userId = await seedParityUser({ + email: `parity-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + const otherUserId = await seedParityUser({ + email: `other-${crypto.randomUUID()}@example.test`, + checkedAt: '9999-12-31T23:59:59.999Z', + }) + + await seedMessageGraph({ + userId, + messageId: 'parity-msg-1', + threadId: 'parity-thread-1', + createdAt: '2026-07-01T12:00:00.000Z', + withAttachment: true, + }) + await seedMessageGraph({ + userId, + messageId: 'parity-msg-2', + threadId: 'parity-thread-2', + createdAt: '2026-07-01T12:01:00.000Z', + }) + await seedOrphanEvent({ + userId, + eventId: 'parity-orphan-1', + createdAt: '2026-07-01T12:02:00.000Z', + }) + await seedMessageGraph({ + userId: otherUserId, + messageId: 'other-msg-1', + threadId: 'other-thread-1', + createdAt: '2026-07-01T12:00:00.000Z', + }) + + const mailbox = rpcFor(userId) + // Warm the DO so the first 1s-bounded mirror RPC is not cold-start timeout. + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const firstNow = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(firstNow) + await expect( + reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + + const d1Counts = await countD1MailboxParity({ db: env.APP_DB, userId }) + expect(await mailbox.countMailbox()).toEqual(d1Counts) + expect(await mailbox.getMessage({ messageId: 'other-msg-1' })).toBeNull() + + const afterMatch = await readParityState(userId) + expect(afterMatch).toMatchObject({ + checkedAt: firstNow.toISOString(), + matchingSince: firstNow.toISOString(), + contentWatermarkAt: expect.any(String), + mismatchCount: 0, + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + messageCursorId: 'parity-msg-2', + orphanCursorId: 'parity-orphan-1', + lastError: null, + }) + if (!afterMatch?.contentWatermarkAt) { + throw new Error('Expected parity content watermark after match.') + } + expect(afterMatch.contentWatermarkAt >= firstNow.toISOString()).toBe(true) + + // Live D1 row after completed cursors: compare mismatches, reopens backfill + // while retaining cursors so the next tick repairs only the new row. + await seedMessageGraph({ + userId, + messageId: 'parity-msg-3', + threadId: 'parity-thread-3', + createdAt: '2026-07-01T12:03:00.000Z', + }) + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + await parkOther(otherUserId) + + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(mismatchNow) + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + const afterMismatch = await readParityState(userId) + expect(afterMismatch).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: expect.any(String), + messagesCompletedAt: null, + orphanEventsCompletedAt: null, + messageCursorId: 'parity-msg-2', + orphanCursorId: 'parity-orphan-1', + checkedAt: mismatchNow.toISOString(), + }) + expect(afterMismatch?.contentWatermarkAt).not.toBeNull() + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + await parkOther(otherUserId) + const rematchNow = new Date('2026-08-01T12:00:00.000Z') + vi.setSystemTime(rematchNow) + await expect( + reconcileMailboxParity({ env, now: rematchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect( + await mailbox.getMessage({ messageId: 'parity-msg-3' }), + ).toMatchObject({ id: 'parity-msg-3' }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) + const afterRematch = await readParityState(userId) + expect(afterRematch).toMatchObject({ + matchingSince: rematchNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + messagesCompletedAt: expect.any(String), + orphanEventsCompletedAt: expect.any(String), + messageCursorId: 'parity-msg-3', + }) + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + await parkOther(otherUserId) + const soakNow = new Date('2026-08-02T12:00:00.000Z') + vi.setSystemTime(soakNow) + await expect( + reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterSoak = await readParityState(userId) + expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) + expect(afterSoak?.contentWatermarkAt).toEqual(expect.any(String)) + expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) + expect(afterSoak?.mismatchCount).toBe(0) + } finally { + vi.useRealTimers() + } +}, 30_000) diff --git a/packages/worker/src/feature-flags/registry.node.test.ts b/packages/worker/src/feature-flags/registry.node.test.ts new file mode 100644 index 0000000000..5361d993d0 --- /dev/null +++ b/packages/worker/src/feature-flags/registry.node.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from 'vitest' +import { + featureFlagDefinitions, + getFeatureFlagDefinition, + isFeatureFlagKey, + measuredFeatureFlagKeys, +} from './registry.ts' + +test('mailbox-read-cutover is registered default-off with email_received success metric', () => { + expect(isFeatureFlagKey('mailbox-read-cutover')).toBe(true) + const definition = getFeatureFlagDefinition('mailbox-read-cutover') + expect(definition).toMatchObject({ + key: 'mailbox-read-cutover', + defaultEnabled: false, + successMetric: { + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + }, + }) + expect(definition.description.length).toBeGreaterThan(0) + expect(measuredFeatureFlagKeys.has('mailbox-read-cutover')).toBe(true) + expect( + featureFlagDefinitions.some((flag) => flag.key === 'mailbox-read-cutover'), + ).toBe(true) +}) diff --git a/packages/worker/src/feature-flags/registry.ts b/packages/worker/src/feature-flags/registry.ts index 563fe37da6..9900341d0b 100644 --- a/packages/worker/src/feature-flags/registry.ts +++ b/packages/worker/src/feature-flags/registry.ts @@ -49,6 +49,19 @@ export const featureFlagDefinitions = [ // exercise the flag system itself (including the "no success metric" // admin notice), not to move a product metric. }, + { + key: 'mailbox-read-cutover', + defaultEnabled: false, + description: + 'Serve owner-scoped user-mail message reads from the per-user Mailbox Durable Object when the account also has continuous D1↔Mailbox parity soak (matching_since ≥ 24h, fresh checked_at, zero mismatches). Default off; fail closed without verified parity. No live reader is switched until a later wiring PR.', + successMetric: { + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + hypothesis: + 'After verified parity soak, reading owner mail from Mailbox should reduce read-path failures that currently surface alongside inbound mail handling.', + }, + }, ] as const satisfies ReadonlyArray export type FeatureFlagKey = (typeof featureFlagDefinitions)[number]['key'] diff --git a/packages/worker/src/feature-flags/service.node.test.ts b/packages/worker/src/feature-flags/service.node.test.ts index 9cf08d2814..68f259fe2f 100644 --- a/packages/worker/src/feature-flags/service.node.test.ts +++ b/packages/worker/src/feature-flags/service.node.test.ts @@ -268,6 +268,7 @@ test('isFeatureEnabled falls back to registry default when no DB state exists', ) await expect(getFeatureFlagsForUser(db, 1)).resolves.toEqual({ 'demo-indicator': false, + 'mailbox-read-cutover': false, }) }) @@ -434,6 +435,7 @@ test('user override wins over global off and global on; clear restores evaluatio await expect(isFeatureEnabled(db, 'demo-indicator', 8)).resolves.toBe(false) await expect(getFeatureFlagsForUser(db, 7)).resolves.toEqual({ 'demo-indicator': true, + 'mailbox-read-cutover': false, }) await setFeatureFlagGlobalState(db, { @@ -465,6 +467,7 @@ test('getFeatureFlagEvaluationsForUser reports assignment sources', async () => await expect(getFeatureFlagEvaluationsForUser(db, 7)).resolves.toEqual({ 'demo-indicator': { enabled: false, source: 'default' }, + 'mailbox-read-cutover': { enabled: false, source: 'default' }, }) await setFeatureFlagGlobalState(db, { @@ -550,7 +553,7 @@ test('listFeatureFlagsForAdmin includes registry flags and stale DB-only keys', }) const listed = await listFeatureFlagsForAdmin(db) - expect(listed).toHaveLength(3) + expect(listed).toHaveLength(4) const demo = listed.find((flag) => flag.key === 'demo-indicator') expect(demo).toMatchObject({ @@ -575,6 +578,22 @@ test('listFeatureFlagsForAdmin includes registry flags and stale DB-only keys', expect(demo?.description).toEqual(expect.any(String)) expect(demo?.description).not.toHaveLength(0) + const mailboxReadCutover = listed.find( + (flag) => flag.key === 'mailbox-read-cutover', + ) + expect(mailboxReadCutover).toMatchObject({ + key: 'mailbox-read-cutover', + stale: false, + defaultEnabled: false, + successMetric: { + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + }, + global: null, + overrides: [], + }) + const retired = listed.find((flag) => flag.key === 'retired-flag') expect(retired).toEqual({ key: 'retired-flag', diff --git a/packages/worker/src/mcp/capabilities/access-control.node.test.ts b/packages/worker/src/mcp/capabilities/access-control.node.test.ts index 402dc3bbd3..47faea4da3 100644 --- a/packages/worker/src/mcp/capabilities/access-control.node.test.ts +++ b/packages/worker/src/mcp/capabilities/access-control.node.test.ts @@ -12,6 +12,7 @@ import { type Capability } from './types.ts' function createFlagMap(enabled: boolean): CallerFeatureFlags { return { 'demo-indicator': enabled, + 'mailbox-read-cutover': false, } } diff --git a/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts index f1f9ab28b9..f7f2318749 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-feature-flag-capabilities.node.test.ts @@ -314,17 +314,33 @@ test('admin feature flag MCP capabilities: list, set, override, and audit wiring const ctx = createAdminCapabilityContext(db) const listResult = await adminFeatureFlagListCapability.handler({}, ctx) - expect(listResult.flags).toEqual([ - expect.objectContaining({ - key: 'demo-indicator', - description: expect.any(String), - defaultEnabled: false, - stale: false, - global: null, - overrides: [], - }), - ]) - expect(listResult.flags[0]?.description).not.toHaveLength(0) + expect(listResult.flags).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: 'demo-indicator', + description: expect.any(String), + defaultEnabled: false, + stale: false, + global: null, + overrides: [], + successMetric: null, + }), + expect.objectContaining({ + key: 'mailbox-read-cutover', + defaultEnabled: false, + stale: false, + successMetric: expect.objectContaining({ + eventType: 'email_received', + measure: 'error_rate', + goal: 'decrease', + }), + }), + ]), + ) + const demoFlag = listResult.flags.find( + (flag) => flag.key === 'demo-indicator', + ) + expect(demoFlag?.description).not.toHaveLength(0) const setResult = await adminFeatureFlagSetCapability.handler( { diff --git a/packages/worker/src/scheduled/scheduled-lanes.node.test.ts b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts new file mode 100644 index 0000000000..dcd75153d4 --- /dev/null +++ b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts @@ -0,0 +1,54 @@ +import { expect, test, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + reconcileMailboxParity: vi.fn(), +})) + +vi.mock('#worker/email/mailbox-reconcile.ts', () => ({ + reconcileMailboxParity: mocks.reconcileMailboxParity, +})) + +const { getScheduledLanes, runScheduledLane, shouldRunMailboxParityCron } = + await import('./scheduled-lanes.ts') + +test('shouldRunMailboxParityCron gates to the top of each UTC hour', () => { + expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:00:30.000Z'))).toBe( + true, + ) + expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:05:00.000Z'))).toBe( + false, + ) + expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:30:00.000Z'))).toBe( + false, + ) +}) + +test('getScheduledLanes includes mailbox_parity only on the hourly gate', () => { + const env = {} as Env + + expect( + getScheduledLanes({ + env, + scheduledAt: new Date('2026-07-05T10:00:30.000Z'), + }), + ).toContain('mailbox_parity') + expect( + getScheduledLanes({ + env, + scheduledAt: new Date('2026-07-05T10:30:00.000Z'), + }), + ).not.toContain('mailbox_parity') +}) + +test('runScheduledLane dispatches mailbox_parity through reconcileMailboxParity', async () => { + const env = {} as Env + const scheduledAt = new Date('2026-07-05T10:00:00.000Z') + mocks.reconcileMailboxParity.mockResolvedValueOnce({ compared: 0 }) + + await runScheduledLane({ env, lane: 'mailbox_parity', scheduledAt }) + + expect(mocks.reconcileMailboxParity).toHaveBeenCalledWith({ + env, + now: scheduledAt, + }) +}) diff --git a/packages/worker/src/scheduled/scheduled-lanes.ts b/packages/worker/src/scheduled/scheduled-lanes.ts index a6922c14d4..1ff5b09dee 100644 --- a/packages/worker/src/scheduled/scheduled-lanes.ts +++ b/packages/worker/src/scheduled/scheduled-lanes.ts @@ -16,6 +16,7 @@ import { shouldRunDrExportCron, shouldRunDrExportWatchdogCron, } from '#worker/dr/exporter.ts' +import { reconcileMailboxParity } from '#worker/email/mailbox-reconcile.ts' import { sweepStaleInboundDeliveries } from '#worker/email/reconcile-inbound-deliveries.ts' import { pruneSystemEmailRetention } from '#worker/email/system-email.ts' import { reconcileD1StorageBytes } from '#worker/entitlements/d1-storage-reconciliation.ts' @@ -32,6 +33,11 @@ import { shouldRunUsageAggregationCron, } from '#worker/usage/aggregate-rollups.ts' +/** Worker cron fires every 5 minutes; run this lane on the top of each UTC hour. */ +export function shouldRunMailboxParityCron(now: Date) { + return now.getUTCMinutes() === 0 +} + export const scheduledLaneNames = [ 'reconcile_artifacts_pushes', 'repo_session_cleanup', @@ -48,6 +54,7 @@ export const scheduledLaneNames = [ 'dr_export', 'dr_export_watchdog', 'job_schedule_watchdog', + 'mailbox_parity', ] as const export type ScheduledLaneName = (typeof scheduledLaneNames)[number] @@ -107,6 +114,9 @@ export function getScheduledLanes(input: { if (shouldRunJobScheduleWatchdogCron(input.scheduledAt)) { lanes.push('job_schedule_watchdog') } + if (shouldRunMailboxParityCron(input.scheduledAt)) { + lanes.push('mailbox_parity') + } return lanes } @@ -183,6 +193,11 @@ export async function runScheduledLane(input: { env: input.env, now: input.scheduledAt, }) + case 'mailbox_parity': + return reconcileMailboxParity({ + env: input.env, + now: input.scheduledAt, + }) default: { const exhaustive: never = input.lane throw new Error(`Unhandled scheduled lane: ${String(exhaustive)}`) diff --git a/packages/worker/src/users-test-schema.ts b/packages/worker/src/users-test-schema.ts index c678e36fad..fd279cb1da 100644 --- a/packages/worker/src/users-test-schema.ts +++ b/packages/worker/src/users-test-schema.ts @@ -74,6 +74,22 @@ const alwaysAdditiveColumns: Record = { active_write_expires_at: { create: 'TEXT' }, d1_storage_bytes: { create: 'INTEGER NOT NULL DEFAULT 0' }, d1_storage_bytes_updated_at: { create: 'TEXT' }, + mailbox_parity_checked_at: { create: 'TEXT' }, + mailbox_parity_matching_since: { create: 'TEXT' }, + mailbox_parity_mismatch_count: { + create: + 'INTEGER NOT NULL DEFAULT 0 CHECK (mailbox_parity_mismatch_count >= 0)', + alter: + 'INTEGER NOT NULL DEFAULT 0 CHECK (mailbox_parity_mismatch_count >= 0)', + }, + mailbox_parity_last_error: { create: 'TEXT' }, + mailbox_parity_content_watermark_at: { create: 'TEXT' }, + mailbox_parity_message_backfill_cursor_created_at: { create: 'TEXT' }, + mailbox_parity_message_backfill_cursor_id: { create: 'TEXT' }, + mailbox_parity_message_backfill_completed_at: { create: 'TEXT' }, + mailbox_parity_orphan_event_backfill_cursor_created_at: { create: 'TEXT' }, + mailbox_parity_orphan_event_backfill_cursor_id: { create: 'TEXT' }, + mailbox_parity_orphan_event_backfill_completed_at: { create: 'TEXT' }, } /** From 3492ad5187d1823e879d228f1144f6b85518383a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 11:51:20 +0000 Subject: [PATCH 11/62] test(email): register Mailbox parity migration Co-authored-by: Kent C. Dodds --- packages/worker/src/index.workers.test.ts | 2 +- tools/migration-ledger.json | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/index.workers.test.ts b/packages/worker/src/index.workers.test.ts index 2014fc5079..460f1f15bb 100644 --- a/packages/worker/src/index.workers.test.ts +++ b/packages/worker/src/index.workers.test.ts @@ -141,7 +141,7 @@ test('scheduled runs gated lanes and passes EMAIL_BLOBS to system-email retentio mocks.shouldRunAuthDenialAlertCron.mockReturnValueOnce(true) mocks.shouldRunEmailDeliveryAlertCron.mockReturnValueOnce(true) mocks.shouldRunJobScheduleWatchdogCron.mockReturnValueOnce(true) - const scheduledTime = Date.parse('2026-07-05T10:00:30.000Z') + const scheduledTime = Date.parse('2026-07-05T10:05:30.000Z') await worker.scheduled?.( createController(scheduledTime), diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 9febdba923..379b71d275 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -511,6 +511,10 @@ { "filename": "0124-entity-sources-external-reconcile-pending.sql", "sha256": "75b93b4c2fa14c6303834877d6a07a42bb054a33b16a55de23c6b56f4b31bd0d" + }, + { + "filename": "0125-mailbox-parity-state.sql", + "sha256": "d5a99f57d8db18943b7184c1c2cc2002c5ef6c20584def71b9860e823b0456cc" } ] } From f03a66f9f8f69f83f77ef81c5a9388803cec36c8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:06:59 +0000 Subject: [PATCH 12/62] fix(email): make Mailbox parity reconciliation resumable Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 142 ++-- .../architecture/feature-flags.md | 8 +- .../migrations/0125-mailbox-parity-state.sql | 44 +- .../worker/src/email/mailbox-parity-phases.ts | 522 ++++++++++++++ .../worker/src/email/mailbox-parity-repo.ts | 120 +++- .../email/mailbox-read-cutover.node.test.ts | 44 ++ .../worker/src/email/mailbox-read-cutover.ts | 4 +- .../src/email/mailbox-reconcile.node.test.ts | 674 ++++++++++++------ .../worker/src/email/mailbox-reconcile.ts | 512 ++++--------- .../email/mailbox-reconcile.workers.test.ts | 131 +++- packages/worker/src/users-test-schema.ts | 9 +- 11 files changed, 1455 insertions(+), 755 deletions(-) create mode 100644 packages/worker/src/email/mailbox-parity-phases.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e8d9a38814..e747ae096d 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -272,12 +272,17 @@ The schema is defined by migrations in `packages/worker/migrations/`: parity/backfill state (migration `0125-mailbox-parity-state.sql`): `mailbox_parity_checked_at`, `mailbox_parity_matching_since`, `mailbox_parity_mismatch_count`, `mailbox_parity_last_error`, - `mailbox_parity_content_watermark_at`, message and orphan-event backfill - cursors/completion timestamps on the user row. Indexed by + `mailbox_parity_content_watermark_at`, durable content-replay window + (`mailbox_parity_content_replay_upper_at` plus `(updated_at, id)` cursor), + message and **all** owner delivery-event backfill cursors/completion + timestamps on the user row. Indexed by `(mailbox_parity_checked_at, stable_user_id)` for oldest-first discovery in the hourly `mailbox_parity` reconcile lane (see - [Mailbox](#durable-objects-mailbox)). Inbound email routing does not - reverse-resolve stable ids at all — it uses the indexed username lookup + [Mailbox](#durable-objects-mailbox)). The same migration adds keyset-friendly + composites on `email_messages(user_id, created_at, id)`, + `email_messages(user_id, updated_at, id)`, and + `email_delivery_events(user_id, created_at, id)`. Inbound email routing does + not reverse-resolve stable ids at all — it uses the indexed username lookup (`findPublicUserIdentityByUsername`). Contextless paths resolve stable ids with one indexed point read on `users.stable_user_id` (for example `findUserAccountByStableUserId`). @@ -805,7 +810,9 @@ mirrors are **not** wired in this slice. Scheduled parity reconcile **is** wired **Scheduled parity reconcile (`mailbox_parity`)** runs as its own queue-isolated scheduled lane (`packages/worker/src/scheduled/scheduled-lanes.ts`, -`packages/worker/src/email/mailbox-reconcile.ts`). The five-minute Worker cron +`packages/worker/src/email/mailbox-reconcile.ts`, +`packages/worker/src/email/mailbox-parity-phases.ts`, +`packages/worker/src/email/mailbox-parity-repo.ts`). The five-minute Worker cron dispatches it only on the top of each UTC hour (`shouldRunMailboxParityCron`); production uses the same per-lane queue isolation as reconcile, retention, and sibling lanes so a slow parity pass cannot consume another lane's budget. @@ -816,27 +823,38 @@ ordered by oldest `mailbox_parity_checked_at`, excluding `system:email`, who have at least one `email_messages` or `email_delivery_events` row. Per-user work is bounded by a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`). +**Account-deletion races:** `loadUserParityState`, `persistUserParityProgress`, +and `rotateCheckedAt` require `deleting_at IS NULL` (zero-row updates are +harmless). Before and after each mirror RPC the lane re-checks +`users.deleting_at`; if deletion started mid-tick it best-effort +`Mailbox.purge()`s and skips further progress for that user. + Per user, the lane: 1. **Content watermark baseline** — on the first backfill attempt, sets `mailbox_parity_content_watermark_at` to the run's `now` and retains it - across incomplete/error ticks so updates during creation/orphan mirroring - stay inside the replay window. + across incomplete/error ticks so updates during creation mirroring stay + inside the replay window. 2. **Bounded initial message backfill** — keyset-pages D1 `email_messages` by - `(created_at, id)` and mirrors each graph via - `mirrorMailboxMessageGraphFromD1` until complete or budget-exhausted. -3. **Bounded orphan-event backfill** — after messages complete, keyset-pages D1 - `email_delivery_events` with `message_id IS NULL` and mirrors each via - `mirrorMailboxOrphanDeliveryEventFromD1`. -4. **Content watermark replay** — after both initial backfills complete, - keyset-replays every owner message with `updated_at` in - `(watermark, current-now]` (classification-like updates that landed after a - graph mirror but before backfill finished). The watermark advances only when - the full window succeeds. -5. **Count compare + AE signals** — owner-scoped D1 counts vs - `Mailbox.countMailbox()` for threads, messages, attachments, and delivery - events. Each comparison emits one `mailbox_parity:` Analytics - Engine row with `double2 = d1Count - doCount`. + `(created_at, id)` (composite index from migration `0125`) and mirrors each + graph via `mirrorMailboxMessageGraphFromD1` until complete or + budget-exhausted. +3. **Bounded delivery-event backfill** — after messages complete, keyset-pages + **every** owner `email_delivery_events` row by `(created_at, id)` — not only + `message_id`-null orphans — and mirrors each via + `mirrorMailboxParityDeliveryEventFromD1`. This repairs events omitted when + per-message graph mirroring truncates the event batch. +4. **Durable content watermark replay** — after both creation phases complete, + opens a frozen window `(watermark, upper]` + (`mailbox_parity_content_replay_upper_at` set once when the window opens; + retained across ticks). Keyset-replays owner messages by `(updated_at, id)` + within that window; the `(updated_at, id)` cursor persists across incomplete + ticks. The watermark advances to `upper` only when the full window succeeds. +5. **Count compare + AE signals** — runs only after the event phase and a + completed content window. Owner-scoped D1 counts vs `Mailbox.countMailbox()` + for threads, messages, attachments, and delivery events. Each comparison + emits one `mailbox_parity:` Analytics Engine row with + `double2 = d1Count - doCount`. Parity progress persists on `users` (migration `0125-mailbox-parity-state.sql`). D1 remains mail authority — the lane never mutates `email_*` rows and does not @@ -845,13 +863,14 @@ flip read authority. **`matching_since` semantics:** records the start of continuous **exact** D1↔Mailbox count parity for phase-3 soak tracking. Set on the first exact compare when null; preserved across successful content replay + exact compare. -Cleared on: initial backfill work in the same tick (`initialBackfilled > 0`), +Cleared on: creation backfill work in the same tick (`creationBackfilled > 0`), incomplete/budget-exhausted ticks, mirror retryable failures, content-replay failure/incomplete, compare mismatch (also increments consecutive -`mailbox_parity_mismatch_count` and reopens bounded creation/orphan backfill -while retaining the content watermark), and compare-path errors -(`rotateCheckedAt` also clears soak so a poison tick cannot preserve a false -window). Resets `mismatch_count` to `0` on exact match. +`mailbox_parity_mismatch_count`, **resets both message and delivery-event +creation cursors and completion markers** for a full historical rescan while +retaining the content watermark and clearing any in-flight replay window), and +compare-path errors (`rotateCheckedAt` also clears soak so a poison tick cannot +preserve a false window). Resets `mismatch_count` to `0` on exact match. **Phase 3 read cutover (prepared, not flipped):** `packages/worker/src/email/mailbox-read-cutover.ts` defines a default-off @@ -859,10 +878,11 @@ window). Resets `mismatch_count` to `0` on exact match. `isMailboxReadCutoverEnabled`. The gate requires the flag **and** per-user parity soak: `mailbox_parity_matching_since` at least 24h old, `mailbox_parity_checked_at` fresh within 2h, -`mailbox_parity_mismatch_count === 0`, and exact `stable_user_id` match. D1 -errors, missing parity state, and an off flag fail closed to D1 reads. No live -reader calls this adapter yet; exposure recording is intentionally omitted until -a live gate site exists (see [Feature flags](./feature-flags.md)). +`mailbox_parity_mismatch_count === 0`, exact `stable_user_id` match, and +`deleting_at IS NULL` (deleting accounts fail closed to D1). D1 errors, missing +parity state, and an off flag fail closed to D1 reads. No live reader calls this +adapter yet; exposure recording is intentionally omitted until a live gate site +exists (see [Feature flags](./feature-flags.md)). **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -895,9 +915,10 @@ batch event repair each use one 1s-bounded RPC; each graph attempt emits at most two AE writes; event repair batches up to 100 events with explicit truncation (newest retained, chronological insertion restored, stable truncation warning) and one batch telemetry outcome. **`system:email` stays in D1 only.** The hourly -queue-isolated `mailbox_parity` lane backfills historical graphs, replays -content-watermark updates, compares owner-scoped D1 vs Mailbox counts, and -persists soak state on `users` (migration `0125`). Inbound lifecycle dual-write, +queue-isolated `mailbox_parity` lane backfills all owner messages and delivery +events, durable content-watermark replays, compares owner-scoped D1 vs Mailbox +counts, persists soak state on `users` (migration `0125`), and re-purges the DO +when account deletion races the lane. Inbound lifecycle dual-write, retention/metadata deletes, and live read cutover are **still not wired**. Live email read/write authority is unchanged — D1 `email_*` tables remain authoritative for all live paths. D1 email rows and existing R2 inventory @@ -918,26 +939,26 @@ delete mirrors remain pending. `mailbox-mutations.ts` + matching DO RPCs (`touchThread`, `updateMessageDelivery`, `setMessageClassification`, `deleteMessageMetadata`, `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and - `mailbox-parity-events.ts`, `mailbox-parity-repo.ts`, and - `mailbox-reconcile.ts`. **Live today (partial):** outbound terminals, - provider delivery-queue outcomes with a message (`recorded`, `duplicate`, - `stale`; `waitUntil`), and user classification - (`service.ts#setEmailMessageClassification`) call + `mailbox-parity-events.ts`, `mailbox-parity-repo.ts`, + `mailbox-parity-phases.ts`, and `mailbox-reconcile.ts`. **Live today + (partial):** outbound terminals, provider delivery-queue outcomes with a + message (`recorded`, `duplicate`, `stale`; `waitUntil`), and user + classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` outcomes automatically (graph repair uses message RPC + one batch event RPC; at most two AE writes per attempt). The hourly `mailbox_parity` scheduled - lane backfills, content-watermark replays, count compares, and soak tracking - on `users` (migration `0125`). **Still pending for phase-2 completion:** - inbound commit/touch/classify dual-write and retention/metadata-delete - mirrors. + lane backfills all owner messages and delivery events, durable + content-watermark replays, count compares, and soak tracking on `users` + (migration `0125`). **Still pending for phase-2 completion:** inbound + commit/touch/classify dual-write and retention/metadata-delete mirrors. 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified (`mailbox_parity_matching_since` ≥ 24h continuous exact counts, fresh - `mailbox_parity_checked_at`, zero `mailbox_parity_mismatch_count`) **and** - the default-off `mailbox-read-cutover` flag is enabled per user. D1 writes - continue. A prepared `getOwnerEmailMessageById` adapter exists but is **not - wired**; no live reader has flipped and no cutover exposures are recorded - yet. + `mailbox_parity_checked_at`, zero `mailbox_parity_mismatch_count`, account + not marked for deletion) **and** the default-off `mailbox-read-cutover` flag + is enabled per user. D1 writes continue. A prepared + `getOwnerEmailMessageById` adapter exists but is **not wired**; no live + reader has flipped and no cutover exposures are recorded yet. 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata to D1; retire dual-write and event/mirror machinery used only for the migration. @@ -945,9 +966,11 @@ delete mirrors remain pending. after verification. No premature schema deletion. The hourly `mailbox_parity` scheduled lane (queue-isolated sibling in -`scheduled-lanes.ts`) owns backfill, content-watermark replay, count parity, and -soak state on `users`. This Mailbox track owns DO storage semantics, alarm -retention, and the inbound durability boundary below. +`scheduled-lanes.ts`) owns backfill of all owner messages and delivery events, +durable content-watermark replay, count parity, soak state on `users`, and +best-effort `Mailbox.purge()` when account deletion races the lane. This Mailbox +track owns DO storage semantics, alarm retention, and the inbound durability +boundary below. ### What stays in D1 @@ -1241,15 +1264,16 @@ Reconcile runs through the registry in `packages/worker/src/scheduled/scheduled-lanes.ts`, alongside repo-session cleanup, system-email retention, general retention, job retention, hourly usage-rollup aggregation, and the hourly `mailbox_parity` lane (top of each UTC -hour only; D1 user discovery, bounded backfill, content-watermark replay, and -count compare — see [Mailbox](#durable-objects-mailbox)). Each production queue -message preserves the existing `scheduled_lane_failed` / D1 lock-contention log -and Sentry context. A handled lane failure is acknowledged and retried by the -next cron tick, matching the old cron semantics. A failed enqueue is reported -and runs through the inline fallback after all sibling enqueue attempts finish; -multiple failed enqueues fall back sequentially to avoid D1 lock contention. -Consumer transport failures retain the configured retry/DLQ behavior. No failure -can abort or mask a sibling invocation. +hour only; D1 user discovery, bounded message + full delivery-event backfill, +durable content-watermark replay, and count compare — see +[Mailbox](#durable-objects-mailbox)). Each production queue message preserves +`scheduled_lane_failed` / D1 lock-contention log and Sentry context. A handled +lane failure is acknowledged and retried by the next cron tick, matching the old +cron semantics. A failed enqueue is reported and runs through the inline +fallback after all sibling enqueue attempts finish; multiple failed enqueues +fall back sequentially to avoid D1 lock contention. Consumer transport failures +retain the configured retry/DLQ behavior. No failure can abort or mask a sibling +invocation. Production note: diff --git a/docs/contributing/architecture/feature-flags.md b/docs/contributing/architecture/feature-flags.md index 627b59b25e..b026c1f661 100644 --- a/docs/contributing/architecture/feature-flags.md +++ b/docs/contributing/architecture/feature-flags.md @@ -83,11 +83,13 @@ parity soak on `users` (migration `0125-mailbox-parity-state.sql`): D1↔Mailbox count parity), - `mailbox_parity_checked_at` fresh within **2 hours** (hourly lane + slack), - `mailbox_parity_mismatch_count === 0`, -- exact `stable_user_id` match between the gate caller and the loaded row. +- exact `stable_user_id` match between the gate caller and the loaded row, +- `deleting_at IS NULL` (accounts marked for deletion fail closed to D1). D1 errors, missing/incomplete parity state, stale `checked_at`, non-zero -mismatch depth, and an off flag all **fail closed** to D1-authoritative reads. -When the gate is on, Mailbox DO read errors propagate with no D1 fallback. +mismatch depth, deleting accounts, and an off flag all **fail closed** to +D1-authoritative reads. When the gate is on, Mailbox DO read errors propagate +with no D1 fallback. The prepared adapter lives in `packages/worker/src/email/mailbox-read-cutover.ts` diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql index 3a2af01586..55a93e2459 100644 --- a/packages/worker/migrations/0125-mailbox-parity-state.sql +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -1,19 +1,18 @@ -- Account-owned Mailbox parity / backfill state for the hourly reconcile lane. -- Low-write columns live on users (not per-message). Creation cursors are --- keyset (created_at, id) so message and message_id-null orphan-event backfill --- can resume across ticks without DO enumeration. content_watermark_at is set --- to the run's now at the start of the first backfill attempt (retained across --- incomplete/error ticks) and bounds a keyset replay of messages with --- updated_at in (watermark, current-now] after both initial backfills complete --- so classification-like updates — including those that land after a graph was --- mirrored but before backfill finished — are remirrored even when counts stay --- equal; the watermark advances only after the full window succeeds. --- matching_since records the start of continuous exact D1 ↔Mailbox count --- parity for soak tracking; it clears on initial backfill work, --- incomplete/budget-exhausted ticks, mirror errors, content-replay --- failure/incomplete, or compare mismatch — but may persist across a --- successful content replay + exact compare. mismatch_count is consecutive --- current mismatch depth and resets to 0 on exact match. +-- keyset (created_at, id) for messages and ALL owner delivery events so backfill +-- can resume across ticks without DO enumeration. The event phase rescans every +-- email_delivery_events row (not only message_id-null) so per-message graph +-- truncation cannot leave DO events behind. content_watermark_at is set to the +-- run's now at the start of the first backfill attempt (retained across +-- incomplete/error ticks). After both creation phases complete, a durable +-- content-replay window (replay_upper_at + replay cursor) keyset-replays +-- messages with updated_at in (watermark, upper]; the watermark advances to +-- upper only when that window finishes. Count compare runs only after the full +-- event phase and a completed content window. Count mismatch clears soak +-- eligibility and resets both creation cursors and completion markers for a +-- full historical rescan (watermark preserved; in-flight replay window cleared). +-- matching_since may persist across a successful content replay + exact compare. ALTER TABLE users ADD COLUMN mailbox_parity_checked_at TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_matching_since TEXT; @@ -21,14 +20,25 @@ ALTER TABLE users ADD COLUMN mailbox_parity_mismatch_count INTEGER NOT NULL DEFA CHECK (mailbox_parity_mismatch_count >= 0); ALTER TABLE users ADD COLUMN mailbox_parity_last_error TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_content_watermark_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_replay_upper_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_replay_cursor_updated_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_content_replay_cursor_id TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_created_at TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_cursor_id TEXT; ALTER TABLE users ADD COLUMN mailbox_parity_message_backfill_completed_at TEXT; -ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_created_at TEXT; -ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_cursor_id TEXT; -ALTER TABLE users ADD COLUMN mailbox_parity_orphan_event_backfill_completed_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_event_backfill_cursor_created_at TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_event_backfill_cursor_id TEXT; +ALTER TABLE users ADD COLUMN mailbox_parity_event_backfill_completed_at TEXT; CREATE INDEX idx_users_mailbox_parity_checked ON users(mailbox_parity_checked_at, stable_user_id); + +-- Keyset-friendly composites for creation scans and content replay. +CREATE INDEX IF NOT EXISTS idx_email_messages_user_created_at_id + ON email_messages(user_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_email_messages_user_updated_at_id + ON email_messages(user_id, updated_at, id); +CREATE INDEX IF NOT EXISTS idx_email_delivery_events_user_created_at_id + ON email_delivery_events(user_id, created_at, id); diff --git a/packages/worker/src/email/mailbox-parity-phases.ts b/packages/worker/src/email/mailbox-parity-phases.ts new file mode 100644 index 0000000000..6a5306f98d --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-phases.ts @@ -0,0 +1,522 @@ +import { + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, + type MailboxLiveMirrorGraphSummary, +} from './mailbox-live-mirror.ts' +import { type MailboxMirrorResult } from './mailbox-mirror.ts' +import { + listContentReplayPage, + listEventBackfillPage, + listMessageBackfillPage, + type MailboxParityContentCursor, + type MailboxParityUserState, +} from './mailbox-parity-repo.ts' + +export const mailboxParityMessagePageSize = 10 +export const mailboxParityEventPageSize = 25 +export const mailboxParityContentPageSize = 10 + +export type MailboxParityPhaseEnv = MailboxLiveMirrorEnv & { + APP_DB: D1Database +} + +export type MailboxParityPhaseResult = { + state: MailboxParityUserState + backfilled: number + budgetExhausted: boolean + retryableFailure: boolean + /** Set when a selected mirror outcome must not advance the cursor. */ + blockedReason: string | null +} + +/** + * Message graph cursor may advance only on mirrored/stale. missing / skipped / + * timeout / error must not advance (selected D1 row was expected). + * `eventsTruncated` is acceptable: the complete owner event phase remirrors + * every delivery event afterward. + */ +export function messageGraphAllowsCursorAdvance( + summary: MailboxLiveMirrorGraphSummary, +): boolean { + switch (summary.message.status) { + case 'mirrored': + case 'stale': + break + case 'missing': + case 'timeout': + case 'error': + case 'skipped': + return false + default: { + const exhaustive: never = summary.message + throw new Error( + `Unhandled graph message status: ${JSON.stringify(exhaustive)}`, + ) + } + } + return !summary.events.some((event) => { + switch (event.result.status) { + case 'mirrored': + case 'stale': + case 'missing': + return false + case 'timeout': + case 'error': + case 'skipped': + return true + default: { + const exhaustive: never = event.result + throw new Error( + `Unhandled graph event status: ${JSON.stringify(exhaustive)}`, + ) + } + } + }) +} + +/** + * Event-phase cursor: mirrored/stale advance; missing advances (D1 row may have + * been deleted concurrently after selection); skipped (incl. unconfigured), + * timeout, and error must not advance. + */ +export function eventMirrorAllowsCursorAdvance( + result: MailboxMirrorResult, +): boolean { + switch (result.status) { + case 'mirrored': + case 'stale': + case 'missing': + return true + case 'timeout': + case 'error': + case 'skipped': + return false + default: { + const exhaustive: never = result + throw new Error( + `Unhandled event mirror status: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +/** + * Load one owner delivery-event projection from D1 and mirror it. + * Thin cohesive wrapper over {@link mirrorMailboxDeliveryEventFromD1}. + */ +export async function mirrorMailboxParityDeliveryEventFromD1(input: { + env: MailboxLiveMirrorEnv + db: D1Database + userId: string + eventId: string +}): Promise { + return mirrorMailboxDeliveryEventFromD1(input) +} + +export function resolveContentReplayUpperBound(input: { + nowIso: string + watermarkAt: string +}): string { + let upperBoundAt = input.nowIso + if (input.watermarkAt === input.nowIso) { + const wallIso = new Date().toISOString() + if (wallIso > upperBoundAt) upperBoundAt = wallIso + } + if (upperBoundAt < input.watermarkAt) return input.watermarkAt + return upperBoundAt +} + +export async function backfillMessagesForUser(input: { + env: MailboxParityPhaseEnv + state: MailboxParityUserState + deadlineMs: number + isDeleting: () => Promise +}): Promise { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listMessageBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.messageCursor, + limit: mailboxParityMessagePageSize, + }) + if (page.length === 0) { + state = { + ...state, + messagesCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + let cursor = state.messageCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + if (await input.isDeleting()) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (await input.isDeleting()) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + if (!messageGraphAllowsCursorAdvance(summary)) { + state = { ...state, messageCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + blockedReason: `message graph ${summary.message.status}`, + deletionStarted: false, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, messageCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } +} + +export async function backfillEventsForUser(input: { + env: MailboxParityPhaseEnv + state: MailboxParityUserState + deadlineMs: number + isDeleting: () => Promise +}): Promise { + let state = input.state + let backfilled = 0 + if (state.messagesCompletedAt == null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + if (state.eventsCompletedAt != null) { + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + while (Date.now() < input.deadlineMs) { + const page = await listEventBackfillPage({ + db: input.env.APP_DB, + userId: state.userId, + cursor: state.eventCursor, + limit: mailboxParityEventPageSize, + }) + if (page.length === 0) { + state = { + ...state, + eventsCompletedAt: new Date().toISOString(), + } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + + let cursor = state.eventCursor + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } + } + if (await input.isDeleting()) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + const result = await mirrorMailboxParityDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + eventId: row.id, + }) + if (await input.isDeleting()) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + } + } + if (!eventMirrorAllowsCursorAdvance(result)) { + state = { ...state, eventCursor: cursor } + return { + state, + backfilled, + budgetExhausted: false, + retryableFailure: true, + blockedReason: + result.status === 'skipped' + ? `event skipped:${result.reason}` + : `event ${result.status}`, + deletionStarted: false, + } + } + cursor = { createdAt: row.created_at, id: row.id } + backfilled += 1 + } + state = { ...state, eventCursor: cursor } + } + + return { + state, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + } +} + +/** + * Durable content replay over (watermark, upper]. Freezes upper when opening a + * window; resumes cursor across ticks; advances watermark only on full success. + */ +export async function replayContentUpdatesForUser(input: { + env: MailboxParityPhaseEnv + state: MailboxParityUserState + nowIso: string + deadlineMs: number + isDeleting: () => Promise +}): Promise< + MailboxParityPhaseResult & { + deletionStarted: boolean + windowComplete: boolean + openedWindow: boolean + } +> { + let state = input.state + const watermarkAt = state.contentWatermarkAt ?? input.nowIso + let openedWindow = false + + if (state.contentReplayUpperAt == null) { + const upperBoundAt = resolveContentReplayUpperBound({ + nowIso: input.nowIso, + watermarkAt, + }) + state = { + ...state, + contentWatermarkAt: watermarkAt, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: null, + } + openedWindow = true + } + + const upperBoundAt = state.contentReplayUpperAt! + let cursor: MailboxParityContentCursor | null = state.contentReplayCursor + let backfilled = 0 + + while (Date.now() < input.deadlineMs) { + const page = await listContentReplayPage({ + db: input.env.APP_DB, + userId: state.userId, + watermarkAt, + upperBoundAt, + cursor, + limit: mailboxParityContentPageSize, + }) + if (page.length === 0) { + return { + state: { + ...state, + contentWatermarkAt: upperBoundAt, + contentReplayUpperAt: null, + contentReplayCursor: null, + }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + windowComplete: true, + openedWindow, + } + } + + for (const row of page) { + if (Date.now() >= input.deadlineMs) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + windowComplete: false, + openedWindow, + } + } + if (await input.isDeleting()) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + windowComplete: false, + openedWindow, + } + } + const summary = await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: state.userId, + messageId: row.id, + }) + if (await input.isDeleting()) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: false, + retryableFailure: false, + blockedReason: null, + deletionStarted: true, + windowComplete: false, + openedWindow, + } + } + if (!messageGraphAllowsCursorAdvance(summary)) { + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: false, + retryableFailure: true, + blockedReason: `content replay ${summary.message.status}`, + deletionStarted: false, + windowComplete: false, + openedWindow, + } + } + cursor = { updatedAt: row.updated_at, id: row.id } + backfilled += 1 + state = { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + } + } + } + + return { + state: { + ...state, + contentReplayUpperAt: upperBoundAt, + contentReplayCursor: cursor, + }, + backfilled, + budgetExhausted: true, + retryableFailure: false, + blockedReason: null, + deletionStarted: false, + windowComplete: false, + openedWindow, + } +} diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts index f41c43a582..6c9c97ccf1 100644 --- a/packages/worker/src/email/mailbox-parity-repo.ts +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -3,8 +3,9 @@ import { type MailboxCountResult } from './mailbox-types.ts' /** * D1 loaders/persisters for the hourly Mailbox parity reconcile lane. - * Creation backfill uses (created_at, id) keysets; content replay uses - * (updated_at, id) within (watermark, now]. + * Creation backfill uses (created_at, id) keysets over messages and all owner + * delivery events; content replay uses a durable (watermark, upper] window with + * an (updated_at, id) cursor. */ export type MailboxParityBackfillCursor = { @@ -22,10 +23,12 @@ export type MailboxParityUserState = { matchingSince: string | null mismatchCount: number contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursor: MailboxParityContentCursor | null messageCursor: MailboxParityBackfillCursor | null messagesCompletedAt: string | null - orphanEventCursor: MailboxParityBackfillCursor | null - orphanEventsCompletedAt: string | null + eventCursor: MailboxParityBackfillCursor | null + eventsCompletedAt: string | null } export type MailboxParityCreatedKeysetRow = { @@ -66,6 +69,22 @@ export async function listUsersForMailboxParity(input: { return result.results ?? [] } +export async function isUserDeleting(input: { + db: D1Database + userId: string +}): Promise { + const row = await input.db + .prepare( + `SELECT deleting_at AS deletingAt + FROM users + WHERE stable_user_id = ? + LIMIT 1`, + ) + .bind(input.userId) + .first<{ deletingAt: string | null }>() + return row == null || row.deletingAt != null +} + export async function countD1MailboxParity(input: { db: D1Database userId: string @@ -114,12 +133,15 @@ export async function loadUserParityState(input: { mailbox_parity_matching_since AS matchingSince, mailbox_parity_mismatch_count AS mismatchCount, mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, + mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, + mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, mailbox_parity_message_backfill_cursor_id AS messageCursorId, mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, - mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, - mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt FROM users WHERE stable_user_id = ? AND deleting_at IS NULL @@ -131,31 +153,44 @@ export async function loadUserParityState(input: { matchingSince: string | null mismatchCount: number | null contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursorUpdatedAt: string | null + contentReplayCursorId: string | null messageCursorCreatedAt: string | null messageCursorId: string | null messagesCompletedAt: string | null - orphanCursorCreatedAt: string | null - orphanCursorId: string | null - orphanEventsCompletedAt: string | null + eventCursorCreatedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null }>() if (!row) return null const messageCursor = row.messageCursorCreatedAt != null && row.messageCursorId != null ? { createdAt: row.messageCursorCreatedAt, id: row.messageCursorId } : null - const orphanEventCursor = - row.orphanCursorCreatedAt != null && row.orphanCursorId != null - ? { createdAt: row.orphanCursorCreatedAt, id: row.orphanCursorId } + const eventCursor = + row.eventCursorCreatedAt != null && row.eventCursorId != null + ? { createdAt: row.eventCursorCreatedAt, id: row.eventCursorId } + : null + const contentReplayCursor = + row.contentReplayCursorUpdatedAt != null && + row.contentReplayCursorId != null + ? { + updatedAt: row.contentReplayCursorUpdatedAt, + id: row.contentReplayCursorId, + } : null return { userId: row.userId, matchingSince: row.matchingSince, mismatchCount: Number(row.mismatchCount ?? 0) || 0, contentWatermarkAt: row.contentWatermarkAt, + contentReplayUpperAt: row.contentReplayUpperAt, + contentReplayCursor, messageCursor, messagesCompletedAt: row.messagesCompletedAt, - orphanEventCursor, - orphanEventsCompletedAt: row.orphanEventsCompletedAt, + eventCursor, + eventsCompletedAt: row.eventsCompletedAt, } } @@ -201,10 +236,10 @@ export async function listMessageBackfillPage(input: { } /** - * Cohesive D1 loader for owner-scoped delivery events with `message_id` null - * (orphan / unbound ledger rows). Keyset-paged for bounded backfill. + * Owner-scoped delivery-event creation keyset (all rows, including those bound + * to messages). Repairs events omitted by per-message graph truncation. */ -export async function listOrphanDeliveryEventBackfillPage(input: { +export async function listEventBackfillPage(input: { db: D1Database userId: string cursor: MailboxParityBackfillCursor | null @@ -217,7 +252,6 @@ export async function listOrphanDeliveryEventBackfillPage(input: { `SELECT id, created_at FROM email_delivery_events WHERE user_id = ? - AND message_id IS NULL ORDER BY created_at ASC, id ASC LIMIT ?`, ) @@ -228,7 +262,6 @@ export async function listOrphanDeliveryEventBackfillPage(input: { `SELECT id, created_at FROM email_delivery_events WHERE user_id = ? - AND message_id IS NULL AND ( created_at > ? OR (created_at = ? AND id > ?) @@ -248,10 +281,8 @@ export async function listOrphanDeliveryEventBackfillPage(input: { } /** - * Keyset page of owner messages updated after `watermarkAt` and at or before - * `upperBoundAt` (run now, or wall time when completing in the baseline run). - * In-tick `cursor` continues within the same window without skipping equal - * `updated_at` rows. + * Keyset page within a frozen content-replay window (watermark, upper]. + * In-tick / durable `cursor` continues without skipping equal `updated_at`. */ export async function listContentReplayPage(input: { db: D1Database @@ -312,13 +343,15 @@ export async function persistUserParityProgress(input: { matchingSince: string | null mismatchCount: number contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursor: MailboxParityContentCursor | null messageCursor: MailboxParityBackfillCursor | null messagesCompletedAt: string | null - orphanEventCursor: MailboxParityBackfillCursor | null - orphanEventsCompletedAt: string | null + eventCursor: MailboxParityBackfillCursor | null + eventsCompletedAt: string | null lastError: string | null -}) { - await input.db +}): Promise<{ wrote: boolean }> { + const result = await input.db .prepare( `UPDATE users SET mailbox_parity_checked_at = ?, @@ -326,13 +359,17 @@ export async function persistUserParityProgress(input: { mailbox_parity_mismatch_count = ?, mailbox_parity_last_error = ?, mailbox_parity_content_watermark_at = ?, + mailbox_parity_content_replay_upper_at = ?, + mailbox_parity_content_replay_cursor_updated_at = ?, + mailbox_parity_content_replay_cursor_id = ?, mailbox_parity_message_backfill_cursor_created_at = ?, mailbox_parity_message_backfill_cursor_id = ?, mailbox_parity_message_backfill_completed_at = ?, - mailbox_parity_orphan_event_backfill_cursor_created_at = ?, - mailbox_parity_orphan_event_backfill_cursor_id = ?, - mailbox_parity_orphan_event_backfill_completed_at = ? - WHERE stable_user_id = ?`, + mailbox_parity_event_backfill_cursor_created_at = ?, + mailbox_parity_event_backfill_cursor_id = ?, + mailbox_parity_event_backfill_completed_at = ? + WHERE stable_user_id = ? + AND deleting_at IS NULL`, ) .bind( input.nowIso, @@ -340,15 +377,19 @@ export async function persistUserParityProgress(input: { input.mismatchCount, input.lastError, input.contentWatermarkAt, + input.contentReplayUpperAt, + input.contentReplayCursor?.updatedAt ?? null, + input.contentReplayCursor?.id ?? null, input.messageCursor?.createdAt ?? null, input.messageCursor?.id ?? null, input.messagesCompletedAt, - input.orphanEventCursor?.createdAt ?? null, - input.orphanEventCursor?.id ?? null, - input.orphanEventsCompletedAt, + input.eventCursor?.createdAt ?? null, + input.eventCursor?.id ?? null, + input.eventsCompletedAt, input.userId, ) .run() + return { wrote: (result.meta?.changes ?? 0) > 0 } } export async function rotateCheckedAt(input: { @@ -356,17 +397,18 @@ export async function rotateCheckedAt(input: { userId: string nowIso: string lastError: string | null -}) { +}): Promise<{ wrote: boolean }> { // Clears matching_since so a poison/error tick cannot preserve a false soak. - // Zero-row updates (deleted/deleting race) are harmless. - await input.db + const result = await input.db .prepare( `UPDATE users SET mailbox_parity_checked_at = ?, mailbox_parity_last_error = ?, mailbox_parity_matching_since = NULL - WHERE stable_user_id = ?`, + WHERE stable_user_id = ? + AND deleting_at IS NULL`, ) .bind(input.nowIso, input.lastError, input.userId) .run() + return { wrote: (result.meta?.changes ?? 0) > 0 } } diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts index 119318cdef..5888ef3d66 100644 --- a/packages/worker/src/email/mailbox-read-cutover.node.test.ts +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -44,9 +44,11 @@ type FlagState = { function createCutoverTestDb(input: { parityByUserId?: Map + deletingUserIds?: ReadonlySet flags?: FlagState }) { const parityByUserId = input.parityByUserId ?? new Map() + const deletingUserIds = input.deletingUserIds ?? new Set() const flags: FlagState = input.flags ?? { overrideEnabled: null, globalEnabled: null, @@ -86,7 +88,9 @@ function createCutoverTestDb(input: { normalized.includes('from users') && normalized.includes('mailbox_parity_matching_since') ) { + expect(normalized).toContain('deleting_at is null') const userId = Number(params[0]) + if (deletingUserIds.has(userId)) return null const row = parityByUserId.get(userId) if (row === 'error') throw new Error('d1 parity read failed') if (row === undefined || row === null) return null @@ -388,6 +392,46 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch ).resolves.toBe(true) }) +test('mailbox-read-cutover keeps deleting accounts on D1 even when otherwise eligible', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const d1Message = baseEmailMessage() + getEmailMessageByIdMock.mockResolvedValue(d1Message) + + const deletingDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + deletingUserIds: new Set([dbUserId]), + }) + await expect( + isMailboxReadCutoverEnabled({ + db: deletingDb, + dbUserId, + stableUserId, + now: nowIso, + }), + ).resolves.toBe(false) + + const getMessage = vi.fn(async () => baseMailboxMessage()) + const { env, idFromName } = fakeMailboxEnv(getMessage) + await expect( + getOwnerEmailMessageById({ + env: { ...env, APP_DB: deletingDb }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + }), + ).resolves.toEqual(d1Message) + expect(getEmailMessageByIdMock).toHaveBeenCalledWith({ + db: deletingDb, + userId: stableUserId, + messageId: 'msg-1', + }) + expect(getMessage).not.toHaveBeenCalled() + expect(idFromName).not.toHaveBeenCalled() +}) + test('mailbox-read-cutover fails closed on invalid now and future parity timestamps', async () => { const dbUserId = 7 const stableUserId = 'user-aaa' diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts index fa3501ebce..c6058323b8 100644 --- a/packages/worker/src/email/mailbox-read-cutover.ts +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -82,11 +82,13 @@ export async function isMailboxReadCutoverEnabled(input: { mailbox_parity_checked_at, mailbox_parity_mismatch_count FROM users - WHERE id = ?`, + WHERE id = ? + AND deleting_at IS NULL`, ) .bind(input.dbUserId) .first() + // Missing row includes deleting accounts (`deleting_at IS NOT NULL`). if (!row) return false if (row.stable_user_id !== input.stableUserId) return false if (row.mailbox_parity_mismatch_count !== 0) return false diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts index 37416204b4..8b639c4787 100644 --- a/packages/worker/src/email/mailbox-reconcile.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.node.test.ts @@ -5,6 +5,10 @@ import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.t import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' import { systemEmailOwnerId } from './email-owner.ts' import type * as MailboxMirrorModule from './mailbox-mirror.ts' +import { + mailboxParityContentPageSize, + mailboxParityMessagePageSize, +} from './mailbox-parity-phases.ts' import { ensureEmailTestSchema } from './test-schema.ts' const mocks = vi.hoisted(() => ({ @@ -38,9 +42,8 @@ vi.mock('./mailbox-client.ts', () => ({ const { countD1MailboxParity, - listOrphanDeliveryEventBackfillPage, + listEventBackfillPage, listUsersForMailboxParity, - mailboxParityMessagePageSize, reconcileMailboxParity, } = await import('./mailbox-reconcile.ts') @@ -81,6 +84,7 @@ async function insertMessage(input: { userId: string id: string createdAt: string + updatedAt?: string }) { await input.db .prepare( @@ -90,23 +94,34 @@ async function insertMessage(input: { ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, 'stored', ?, ?)`, ) - .bind(input.id, input.userId, input.createdAt, input.createdAt) + .bind( + input.id, + input.userId, + input.createdAt, + input.updatedAt ?? input.createdAt, + ) .run() } -async function insertOrphanEvent(input: { +async function insertEvent(input: { db: D1Database userId: string id: string createdAt: string + messageId?: string | null }) { await input.db .prepare( `INSERT INTO email_delivery_events ( id, message_id, user_id, event_type, provider, detail_json, created_at - ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) VALUES (?, ?, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind( + input.id, + input.messageId === undefined ? null : input.messageId, + input.userId, + input.createdAt, ) - .bind(input.id, input.userId, input.createdAt) .run() } @@ -130,12 +145,15 @@ async function readParityRow(db: D1Database, userId: string) { mailbox_parity_mismatch_count AS mismatchCount, mailbox_parity_last_error AS lastError, mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, + mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, + mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, mailbox_parity_message_backfill_cursor_id AS messageCursorId, mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_orphan_event_backfill_cursor_created_at AS orphanCursorCreatedAt, - mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, - mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt + mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt FROM users WHERE stable_user_id = ?`, ) @@ -146,21 +164,24 @@ async function readParityRow(db: D1Database, userId: string) { mismatchCount: number lastError: string | null contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursorUpdatedAt: string | null + contentReplayCursorId: string | null messageCursorCreatedAt: string | null messageCursorId: string | null messagesCompletedAt: string | null - orphanCursorCreatedAt: string | null - orphanCursorId: string | null - orphanEventsCompletedAt: string | null + eventCursorCreatedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null }>() } -function successGraph(messageId: string) { +function successGraph(messageId: string, eventsTruncated = false) { return { messageId, message: { status: 'mirrored' as const }, events: [], - eventsTruncated: false, + eventsTruncated, } } @@ -176,9 +197,20 @@ function mockDoCounts(counts: { }) mocks.mailboxRpc.mockImplementation(() => ({ countMailbox: vi.fn(async () => counts), + purge: vi.fn(async () => ({ ok: true as const })), })) } +function parityEnv(db: D1Database) { + return { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } +} + test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { consoleWarn.mockImplementation(() => {}) const { db } = await createParityDb() @@ -212,12 +244,19 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, }) } - await insertOrphanEvent({ + await insertEvent({ db, userId: ownerA, - id: 'orphan-1', + id: 'evt-bound-1', + messageId: messageIds[0]!, createdAt: '2026-07-02T00:00:00.000Z', }) + await insertEvent({ + db, + userId: ownerA, + id: 'evt-orphan-1', + createdAt: '2026-07-02T00:00:01.000Z', + }) await insertMessage({ db, userId: ownerB, @@ -230,27 +269,27 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou id: 'sys-msg-1', createdAt: '2026-07-01T00:00:00.000Z', }) - await insertOrphanEvent({ + await insertEvent({ db, userId: deleting, - id: 'del-orphan', + id: 'del-evt', createdAt: '2026-07-01T00:00:00.000Z', }) - // NULL checked_at sorts first under ORDER BY checked_at ASC (index-friendly). const discovered = await listUsersForMailboxParity({ db, limit: 10 }) expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) expect(discovered.map((row) => row.userId)).not.toContain(deleting) - const orphanPage = await listOrphanDeliveryEventBackfillPage({ + const eventPage = await listEventBackfillPage({ db, userId: ownerA, cursor: null, limit: 10, }) - expect(orphanPage).toEqual([ - { id: 'orphan-1', created_at: '2026-07-02T00:00:00.000Z' }, + expect(eventPage.map((row) => row.id)).toEqual([ + 'evt-bound-1', + 'evt-orphan-1', ]) await db @@ -266,20 +305,12 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou '2026-07-01T00:00:00.000Z', ) .run() - await db - .prepare( - `INSERT INTO email_attachments ( - id, message_id, content_type, size, storage_kind, created_at - ) VALUES ('att-1', ?, 'text/plain', 1, 'unavailable', ?)`, - ) - .bind(messageIds[0]!, '2026-07-01T00:00:00.000Z') - .run() await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ threads: 1, messages: mailboxParityMessagePageSize + 1, - attachments: 1, - deliveryEvents: 1, + attachments: 0, + deliveryEvents: 2, }) let graphCalls = 0 @@ -306,17 +337,8 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou attachments: 0, deliveryEvents: 0, }) - mocks.recordMailboxParityEvent.mockClear() - const env = { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } - - // Seed a false soak that must clear on mirror timeout / backfill work. + const env = parityEnv(db) await db .prepare( `UPDATE users @@ -344,30 +366,22 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou expect(consoleWarn).toHaveBeenCalledWith( 'mailbox-parity-message-backfill-retryable', ownerA, - 'error-or-timeout', + 'message graph timeout', ) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.every( - (call) => call[0].userId === ownerA, - ), - ).toBe(true) - expect(mocks.mailboxRpc).not.toHaveBeenCalled() const afterFirst = await readParityRow(db, ownerA) expect(afterFirst).toMatchObject({ checkedAt: now.toISOString(), messageCursorId: messageIds[mailboxParityMessagePageSize - 2], messagesCompletedAt: null, - orphanEventsCompletedAt: null, - // Baseline is persisted at first backfill attempt start and retained. + eventsCompletedAt: null, contentWatermarkAt: now.toISOString(), matchingSince: null, mismatchCount: 2, - lastError: 'message backfill mirror error or timeout', + lastError: 'message backfill message graph timeout', }) - // Finish backfill; compare mismatches → reopen completed_at, retain cursors, - // increment consecutive mismatch count, keep matching_since cleared. + // Finish creation; count mismatch resets BOTH cursors + completion markers. graphCalls = 0 mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), @@ -384,16 +398,18 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou }) expect(mismatch).toEqual({ scanned: 1, - backfilled: 3, + backfilled: 4, compared: 1, matched: 0, mismatched: 1, failed: 0, }) expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'orphan-1' }), + expect.objectContaining({ userId: ownerA, eventId: 'evt-bound-1' }), + ) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'evt-orphan-1' }), ) - expect(mocks.recordMailboxParityEvent).toHaveBeenCalledTimes(4) const afterMismatch = await readParityRow(db, ownerA) expect(afterMismatch).toMatchObject({ @@ -401,14 +417,16 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou matchingSince: null, mismatchCount: 3, contentWatermarkAt: mismatchNow.toISOString(), + contentReplayUpperAt: null, + contentReplayCursorId: null, messagesCompletedAt: null, - orphanEventsCompletedAt: null, - messageCursorId: messageIds[mailboxParityMessagePageSize], - orphanCursorId: 'orphan-1', + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, lastError: null, }) - // Live new row after retained cursor must be repaired on the reopen pass. + // Full historical rescan after cursor reset, including a live new row. await insertMessage({ db, userId: ownerA, @@ -420,8 +438,8 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou mockDoCounts({ threads: 1, messages: mailboxParityMessagePageSize + 2, - attachments: 1, - deliveryEvents: 1, + attachments: 0, + deliveryEvents: 2, }) await parkUser(db, ownerB) const rematchNow = new Date('2026-08-01T12:00:00.000Z') @@ -432,7 +450,7 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou }) expect(rematch).toEqual({ scanned: 1, - backfilled: 1, + backfilled: mailboxParityMessagePageSize + 4, compared: 1, matched: 1, mismatched: 0, @@ -447,12 +465,11 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou mismatchCount: 0, contentWatermarkAt: rematchNow.toISOString(), messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), messageCursorId: 'msg-live-new', + eventCursorId: 'evt-orphan-1', }) - // 24h soak preservation: exact completed compare with empty content window - // advances the watermark and keeps matching_since. await parkUser(db, ownerB) const soakNow = new Date('2026-08-02T12:00:00.000Z') await expect( @@ -468,10 +485,8 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou const afterSoak = await readParityRow(db, ownerA) expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) - expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) expect(afterSoak?.mismatchCount).toBe(0) - // Compare-path error rotates checked_at and clears matching_since. consoleWarn.mockClear() mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( new Error('count boom\nwith\tcontrol'), @@ -485,17 +500,6 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou expect(afterError?.checkedAt).toBe(errorNow.toISOString()) expect(afterError?.lastError).toBe('count boom with control') expect(afterError?.matchingSince).toBeNull() - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-reconcile-user-failed', - ownerA, - expect.any(Error), - ) - - // Discovery stays D1-only: mailboxRpc is never used to enumerate candidates. - const rpcCallsBefore = mocks.mailboxRpc.mock.calls.length - const discoveryOnly = await listUsersForMailboxParity({ db, limit: 5 }) - expect(discoveryOnly.length).toBeGreaterThan(0) - expect(mocks.mailboxRpc).toHaveBeenCalledTimes(rpcCallsBefore) }) test('mailbox parity content watermark repairs equal-count updates and soak', async () => { @@ -533,52 +537,24 @@ test('mailbox parity content watermark repairs equal-count updates and soak', as attachments: 0, deliveryEvents: 0, }) + const env = parityEnv(db) - const env = { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } - - // Baseline at first-attempt start; empty same-run window; then compare. await expect( reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 2, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - const afterBaseline = await readParityRow(db, owner) - expect(afterBaseline).toMatchObject({ - matchingSince: now.toISOString(), - contentWatermarkAt: now.toISOString(), - mismatchCount: 0, - }) + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + now.toISOString(), + ) - // Classification-like update: same counts, newer updated_at past watermark. const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' await db .prepare( `UPDATE email_messages SET classification = 'quarantined', updated_at = ? - WHERE id = 'msg-a'`, - ) - .bind(sharedUpdatedAt) - .run() - await db - .prepare( - `UPDATE email_messages - SET classification = 'quarantined', updated_at = ? - WHERE id = 'msg-b'`, + WHERE id IN ('msg-a', 'msg-b')`, ) .bind(sharedUpdatedAt) .run() - mocks.mirrorMailboxMessageGraphFromD1.mockClear() const repairNow = new Date('2026-08-01T11:00:00.000Z') vi.setSystemTime(repairNow) @@ -592,104 +568,15 @@ test('mailbox parity content watermark repairs equal-count updates and soak', as mismatched: 0, failed: 0, }) - const replayedIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ) - expect(replayedIds).toEqual(['msg-a', 'msg-b']) - const afterRepair = await readParityRow(db, owner) - // Successful content sweep + exact compare keeps the soak window. - expect(afterRepair?.matchingSince).toBe(now.toISOString()) - expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) - expect(afterRepair?.mismatchCount).toBe(0) - - // Equal-timestamp rows after the advanced watermark are both replayed. - const equalStamp = '2026-08-01T11:15:00.000Z' - await db - .prepare( - `UPDATE email_messages SET updated_at = ? WHERE id IN ('msg-a', 'msg-b')`, - ) - .bind(equalStamp) - .run() - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - const equalNow = new Date('2026-08-01T12:00:00.000Z') - vi.setSystemTime(equalNow) - await expect( - reconcileMailboxParity({ env, now: equalNow, batchSize: 1 }), - ).resolves.toMatchObject({ - backfilled: 2, - compared: 1, - matched: 1, - failed: 0, - }) expect( mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( (call) => call[0].messageId as string, ), ).toEqual(['msg-a', 'msg-b']) - expect((await readParityRow(db, owner))?.matchingSince).toBe( - now.toISOString(), - ) - expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( - equalNow.toISOString(), - ) - - // Failure mid-window: watermark does not advance; matching_since clears. - await db - .prepare( - `UPDATE email_messages - SET updated_at = '2026-08-01T12:30:00.000Z' - WHERE id IN ('msg-a', 'msg-b')`, - ) - .run() - await db - .prepare( - `UPDATE users - SET mailbox_parity_matching_since = ? - WHERE stable_user_id = ?`, - ) - .bind(now.toISOString(), owner) - .run() - let contentCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - contentCalls += 1 - if (contentCalls === 1) { - return { - messageId: input.messageId, - message: { status: 'timeout' as const }, - events: [], - eventsTruncated: false, - } - } - return successGraph(input.messageId) - }, - ) - consoleWarn.mockClear() - const failNow = new Date('2026-08-01T13:00:00.000Z') - vi.setSystemTime(failNow) - await expect( - reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 0, - matched: 0, - mismatched: 0, - failed: 1, - }) - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-content-replay-retryable', - owner, - 'error-or-timeout', - ) - const afterFail = await readParityRow(db, owner) - expect(afterFail).toMatchObject({ - contentWatermarkAt: equalNow.toISOString(), - matchingSince: null, - checkedAt: failNow.toISOString(), - lastError: 'content replay mirror error or timeout', - mismatchCount: 0, - }) + const afterRepair = await readParityRow(db, owner) + expect(afterRepair?.matchingSince).toBe(now.toISOString()) + expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) + expect(afterRepair?.contentReplayUpperAt).toBeNull() } finally { vi.useRealTimers() } @@ -718,11 +605,9 @@ test('mailbox parity replays message updated during initial backfill', async () id: 'msg-late', createdAt: '2026-07-01T00:00:01.000Z', }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => { if (input.messageId === 'msg-early') { - // Concurrent classification-like update after the graph mirror. vi.setSystemTime(midUpdateAt) await db .prepare( @@ -745,17 +630,12 @@ test('mailbox parity replays message updated during initial backfill', async () attachments: 0, deliveryEvents: 0, }) - - const env = { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), ).resolves.toEqual({ scanned: 1, backfilled: 3, @@ -764,20 +644,356 @@ test('mailbox parity replays message updated during initial backfill', async () mismatched: 0, failed: 0, }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-early', 'msg-late', 'msg-early']) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + midUpdateAt.toISOString(), + ) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity content replay resumes durable upper and cursor across ticks', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'f'.repeat(64) + const baseline = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(baseline) - const mirroredIds = mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, + try { + await insertUser({ db, userId: owner, checkedAt: null }) + const messageCount = mailboxParityContentPageSize + 2 + for (let index = 0; index < messageCount; index += 1) { + await insertMessage({ + db, + userId: owner, + id: `msg-${String(index).padStart(2, '0')}`, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), ) - expect(mirroredIds).toEqual(['msg-early', 'msg-late', 'msg-early']) - const after = await readParityRow(db, owner) - expect(after).toMatchObject({ - matchingSince: now.toISOString(), - contentWatermarkAt: midUpdateAt.toISOString(), - mismatchCount: 0, - messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: messageCount, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await reconcileMailboxParity({ env, now: baseline, batchSize: 1 }) + + const updateAt = '2026-08-01T10:30:00.000Z' + await db + .prepare(`UPDATE email_messages SET updated_at = ? WHERE user_id = ?`) + .bind(updateAt, owner) + .run() + + const replayNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(replayNow) + let contentCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + contentCalls += 1 + if (contentCalls === mailboxParityContentPageSize) { + // Exhaust budget mid-window by freezing time past deadline. + vi.setSystemTime(new Date(replayNow.getTime() + 20_000)) + } + return successGraph(input.messageId) + }, + ) + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 0, + failed: 0, + backfilled: mailboxParityContentPageSize, + }) + const mid = await readParityRow(db, owner) + expect(mid).toMatchObject({ + contentWatermarkAt: baseline.toISOString(), + contentReplayUpperAt: replayNow.toISOString(), + contentReplayCursorId: `msg-${String(mailboxParityContentPageSize - 1).padStart(2, '0')}`, + matchingSince: null, + }) + + vi.setSystemTime(replayNow) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 1, + matched: 1, + backfilled: 2, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual([ + `msg-${String(mailboxParityContentPageSize).padStart(2, '0')}`, + `msg-${String(mailboxParityContentPageSize + 1).padStart(2, '0')}`, + ]) + const done = await readParityRow(db, owner) + expect(done).toMatchObject({ + contentWatermarkAt: replayNow.toISOString(), + contentReplayUpperAt: null, + contentReplayCursorId: null, + // Soak was cleared on the incomplete tick; exact compare starts a new window. + matchingSince: replayNow.toISOString(), }) } finally { vi.useRealTimers() } }) + +test('mailbox parity missing and unconfigured do not advance creation cursors', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'g'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-2', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'msg-1', + message: { status: 'missing' as const }, + events: [], + eventsTruncated: false, + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 1, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, backfilled: 0, compared: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + messageCursorId: null, + messagesCompletedAt: null, + contentWatermarkAt: now.toISOString(), + lastError: 'message backfill message graph missing', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValueOnce({ + status: 'skipped', + reason: 'mailbox-unconfigured', + }) + await expect( + reconcileMailboxParity({ + env, + now: new Date('2026-08-01T11:00:00.000Z'), + batchSize: 1, + }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterUnconfigured = await readParityRow(db, owner) + expect(afterUnconfigured).toMatchObject({ + messagesCompletedAt: expect.any(String), + eventCursorId: null, + eventsCompletedAt: null, + lastError: 'event backfill event skipped:mailbox-unconfigured', + }) +}) + +test('mailbox parity eventsTruncated still completes via full event phase', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'h'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + for (let index = 0; index < 3; index += 1) { + await insertEvent({ + db, + userId: owner, + id: `evt-${index}`, + messageId: 'msg-1', + createdAt: `2026-07-01T00:01:0${index}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue( + successGraph('msg-1', true), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 3, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 4, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledTimes(3) +}) + +test('mailbox parity event missing advances cursor while unconfigured does not', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'j'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-gone', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-ok', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1 + .mockResolvedValueOnce({ status: 'missing' }) + .mockResolvedValueOnce({ status: 'mirrored' }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 2, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toMatchObject({ matched: 1, failed: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + eventCursorId: 'evt-ok', + eventsCompletedAt: expect.any(String), + }) +}) + +test('mailbox parity deletion race purges Mailbox and skips parity writes', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'i'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + })), + purge, + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + try { + return { ok: true as const, value: await promise } + } catch { + return { ok: false as const, result: { status: 'timeout' as const } } + } + }, + ) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { + await db + .prepare(`UPDATE users SET deleting_at = ? WHERE stable_user_id = ?`) + .bind(now.toISOString(), owner) + .run() + return successGraph('msg-1') + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const row = await readParityRow(db, owner) + // Baseline may have been written before deletion; no completion/compare state. + expect(row?.messagesCompletedAt).toBeNull() + expect(row?.matchingSince).toBeNull() + expect(row?.eventsCompletedAt).toBeNull() +}) diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index a8d3dcc097..d2c9943930 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -1,30 +1,31 @@ import { truncateToUtf8Bytes } from '@kody-internal/shared/backup-restore-safety.ts' import { mailboxRpc } from './mailbox-client.ts' -import { - mirrorMailboxDeliveryEventFromD1, - mirrorMailboxMessageGraphFromD1, - type MailboxLiveMirrorEnv, - type MailboxLiveMirrorGraphSummary, -} from './mailbox-live-mirror.ts' +import { type MailboxLiveMirrorEnv } from './mailbox-live-mirror.ts' import { awaitMailboxMirrorRpc, mailboxMirrorRpcTimeoutMs, - type MailboxMirrorResult, } from './mailbox-mirror.ts' import { recordMailboxParityEvent, type MailboxParityOperation, } from './mailbox-parity-events.ts' +import { + backfillEventsForUser, + backfillMessagesForUser, + mailboxParityContentPageSize, + mailboxParityEventPageSize, + mailboxParityMessagePageSize, + mirrorMailboxParityDeliveryEventFromD1, + replayContentUpdatesForUser, +} from './mailbox-parity-phases.ts' import { countD1MailboxParity, - listContentReplayPage, - listMessageBackfillPage, - listOrphanDeliveryEventBackfillPage, + isUserDeleting, + listEventBackfillPage, listUsersForMailboxParity, loadUserParityState, persistUserParityProgress, rotateCheckedAt, - type MailboxParityContentCursor, type MailboxParityUserState, } from './mailbox-parity-repo.ts' import { type MailboxCountResult } from './mailbox-types.ts' @@ -33,20 +34,24 @@ import { type MailboxCountResult } from './mailbox-types.ts' * Bounded hourly Mailbox backfill + count-parity reconciler. * * Discovers a small oldest-first page of non-deleting D1 mail owners (no DO - * enumeration), mirrors message graphs then message_id-null orphan delivery - * events through existing live-mirror helpers, then keyset-replays messages - * updated after the content watermark before comparing owner-scoped D1 counts - * with `countMailbox`. D1 remains mail authority; this lane never mutates - * email_* rows and does not flip read authority. + * enumeration), mirrors message graphs then every owner delivery event through + * existing live-mirror helpers (repairing graph truncation), durably + * keyset-replays messages updated after the content watermark, then compares + * owner-scoped D1 counts with `countMailbox`. D1 remains mail authority; this + * lane never mutates email_* rows and does not flip read authority. */ export const mailboxParityUserBatchSize = 8 -export const mailboxParityMessagePageSize = 10 -export const mailboxParityOrphanEventPageSize = 25 -export const mailboxParityContentPageSize = 10 export const mailboxParityTimeBudgetMs = 10_000 export const mailboxParityLastErrorMaxBytes = 512 +export { + mailboxParityContentPageSize, + mailboxParityEventPageSize, + mailboxParityMessagePageSize, + mirrorMailboxParityDeliveryEventFromD1, +} + export type MailboxParityReconcileEnv = MailboxLiveMirrorEnv & { APP_DB: D1Database } @@ -62,19 +67,10 @@ export type MailboxParityReconcileMetrics = { export { countD1MailboxParity, - listOrphanDeliveryEventBackfillPage, + listEventBackfillPage, listUsersForMailboxParity, } -function isRetryableMirrorResult(result: MailboxMirrorResult): boolean { - return result.status === 'error' || result.status === 'timeout' -} - -function graphMirrorSucceeded(summary: MailboxLiveMirrorGraphSummary): boolean { - if (isRetryableMirrorResult(summary.message)) return false - return !summary.events.some((event) => isRetryableMirrorResult(event.result)) -} - function safeParityErrorText(error: unknown): string { const raw = error instanceof Error ? error.message : String(error) let cleaned = '' @@ -89,17 +85,18 @@ function safeParityErrorText(error: unknown): string { ) } -/** - * Load one orphan delivery-event projection from D1 and mirror it. - * Thin cohesive wrapper over {@link mirrorMailboxDeliveryEventFromD1}. - */ -export async function mirrorMailboxOrphanDeliveryEventFromD1(input: { - env: MailboxLiveMirrorEnv - db: D1Database +async function purgeMailboxBestEffort(input: { + env: MailboxParityReconcileEnv userId: string - eventId: string -}): Promise { - return mirrorMailboxDeliveryEventFromD1(input) +}) { + try { + await awaitMailboxMirrorRpc( + mailboxRpc({ env: input.env, userId: input.userId }).purge(), + mailboxMirrorRpcTimeoutMs, + ) + } catch (error) { + console.warn('mailbox-parity-purge-on-delete-failed', input.userId, error) + } } function emitCountComparisons(input: { @@ -153,199 +150,6 @@ function emitCountComparisons(input: { return { matched } } -async function backfillMessagesForUser(input: { - env: MailboxParityReconcileEnv - state: MailboxParityUserState - deadlineMs: number -}): Promise<{ - state: MailboxParityUserState - backfilled: number - budgetExhausted: boolean - retryableFailure: boolean -}> { - let state = input.state - let backfilled = 0 - if (state.messagesCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - while (Date.now() < input.deadlineMs) { - const page = await listMessageBackfillPage({ - db: input.env.APP_DB, - userId: state.userId, - cursor: state.messageCursor, - limit: mailboxParityMessagePageSize, - }) - if (page.length === 0) { - state = { - ...state, - messagesCompletedAt: new Date().toISOString(), - } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - let cursor = state.messageCursor - for (const row of page) { - if (Date.now() >= input.deadlineMs) { - state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } - } - const summary = await mirrorMailboxMessageGraphFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: state.userId, - messageId: row.id, - }) - if (!graphMirrorSucceeded(summary)) { - state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: true, - } - } - cursor = { createdAt: row.created_at, id: row.id } - backfilled += 1 - } - state = { ...state, messageCursor: cursor } - } - - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } -} - -async function backfillOrphanEventsForUser(input: { - env: MailboxParityReconcileEnv - state: MailboxParityUserState - deadlineMs: number -}): Promise<{ - state: MailboxParityUserState - backfilled: number - budgetExhausted: boolean - retryableFailure: boolean -}> { - let state = input.state - let backfilled = 0 - if (state.messagesCompletedAt == null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - if (state.orphanEventsCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - while (Date.now() < input.deadlineMs) { - const page = await listOrphanDeliveryEventBackfillPage({ - db: input.env.APP_DB, - userId: state.userId, - cursor: state.orphanEventCursor, - limit: mailboxParityOrphanEventPageSize, - }) - if (page.length === 0) { - state = { - ...state, - orphanEventsCompletedAt: new Date().toISOString(), - } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - } - } - - let cursor = state.orphanEventCursor - for (const row of page) { - if (Date.now() >= input.deadlineMs) { - state = { ...state, orphanEventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } - } - const result = await mirrorMailboxOrphanDeliveryEventFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: state.userId, - eventId: row.id, - }) - if (isRetryableMirrorResult(result)) { - state = { ...state, orphanEventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: true, - } - } - cursor = { createdAt: row.created_at, id: row.id } - backfilled += 1 - } - state = { ...state, orphanEventCursor: cursor } - } - - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - } -} - -/** - * Baseline is established before the first creation scan. When it still equals - * this run's nowIso, sample wall time so same-run updates after the baseline - * are included; otherwise the run's nowIso is the upper bound. - */ -function resolveContentReplayUpperBound(input: { - nowIso: string - watermarkAt: string -}): string { - let upperBoundAt = input.nowIso - if (input.watermarkAt === input.nowIso) { - const wallIso = new Date().toISOString() - if (wallIso > upperBoundAt) upperBoundAt = wallIso - } - if (upperBoundAt < input.watermarkAt) return input.watermarkAt - return upperBoundAt -} - -/** - * Persist the content watermark at the start of the first backfill attempt so - * updates during creation/orphan mirroring cannot fall before a completion-time - * baseline. Retained across incomplete/error ticks. - */ async function ensureContentWatermarkBaseline(input: { db: D1Database state: MailboxParityUserState @@ -363,100 +167,17 @@ async function ensureContentWatermarkBaseline(input: { matchingSince: state.matchingSince, mismatchCount: state.mismatchCount, contentWatermarkAt: state.contentWatermarkAt, + contentReplayUpperAt: state.contentReplayUpperAt, + contentReplayCursor: state.contentReplayCursor, messageCursor: state.messageCursor, messagesCompletedAt: state.messagesCompletedAt, - orphanEventCursor: state.orphanEventCursor, - orphanEventsCompletedAt: state.orphanEventsCompletedAt, + eventCursor: state.eventCursor, + eventsCompletedAt: state.eventsCompletedAt, lastError: null, }) return state } -/** - * After initial creation/orphan backfill completes: keyset-replay every owner - * message with updated_at in (watermark, current-now]. Watermark advances only - * when the entire window succeeds; in-tick cursor avoids skipping equal - * timestamps. Always replay — never treat a fresh baseline as an empty skip. - */ -async function replayContentUpdatesForUser(input: { - env: MailboxParityReconcileEnv - state: MailboxParityUserState - nowIso: string - deadlineMs: number -}): Promise<{ - state: MailboxParityUserState - backfilled: number - budgetExhausted: boolean - retryableFailure: boolean - windowComplete: boolean -}> { - const state = input.state - const watermarkAt = state.contentWatermarkAt ?? input.nowIso - const upperBoundAt = resolveContentReplayUpperBound({ - nowIso: input.nowIso, - watermarkAt, - }) - let cursor: MailboxParityContentCursor | null = null - let backfilled = 0 - - while (Date.now() < input.deadlineMs) { - const page = await listContentReplayPage({ - db: input.env.APP_DB, - userId: state.userId, - watermarkAt, - upperBoundAt, - cursor, - limit: mailboxParityContentPageSize, - }) - if (page.length === 0) { - return { - state: { ...state, contentWatermarkAt: upperBoundAt }, - backfilled, - budgetExhausted: false, - retryableFailure: false, - windowComplete: true, - } - } - - for (const row of page) { - if (Date.now() >= input.deadlineMs) { - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - windowComplete: false, - } - } - const summary = await mirrorMailboxMessageGraphFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: state.userId, - messageId: row.id, - }) - if (!graphMirrorSucceeded(summary)) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: true, - windowComplete: false, - } - } - cursor = { updatedAt: row.updated_at, id: row.id } - backfilled += 1 - } - } - - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - windowComplete: false, - } -} - async function compareParityForUser(input: { env: MailboxParityReconcileEnv state: MailboxParityUserState @@ -494,16 +215,19 @@ async function compareParityForUser(input: { matched: true, } } - // Reopen bounded creation/orphan backfill after completed cursors so later - // live rows past the retained keyset are mirrored on the next ticks; keep - // the content watermark so classification updates remain covered. + // Full historical rescan: clear creation cursors + completion markers. + // Preserve content watermark; drop any in-flight replay window; clear soak. return { state: { ...input.state, matchingSince: null, mismatchCount: input.state.mismatchCount + 1, + messageCursor: null, messagesCompletedAt: null, - orphanEventsCompletedAt: null, + eventCursor: null, + eventsCompletedAt: null, + contentReplayUpperAt: null, + contentReplayCursor: null, }, matched: false, } @@ -539,25 +263,50 @@ async function reconcileOneUser(input: { let state = loaded let backfilled = 0 - let initialBackfilled = 0 + let creationBackfilled = 0 + + const isDeleting = () => + isUserDeleting({ db: input.env.APP_DB, userId: state.userId }) + + const stopForDeletion = async () => { + await purgeMailboxBestEffort({ + env: input.env, + userId: state.userId, + }) + return { + backfilled, + compared: false, + matched: false, + mismatched: false, + failed: false, + budgetExhausted: false, + } + } - const persistIncomplete = async (inputPersist: { + const persistProgress = async (inputPersist: { + matchingSince: string | null lastError: string | null }) => { - await persistUserParityProgress({ + const { wrote } = await persistUserParityProgress({ db: input.env.APP_DB, userId: state.userId, nowIso: input.nowIso, - // Soak cannot survive incomplete/error/backfill ticks. - matchingSince: null, + matchingSince: inputPersist.matchingSince, mismatchCount: state.mismatchCount, contentWatermarkAt: state.contentWatermarkAt, + contentReplayUpperAt: state.contentReplayUpperAt, + contentReplayCursor: state.contentReplayCursor, messageCursor: state.messageCursor, messagesCompletedAt: state.messagesCompletedAt, - orphanEventCursor: state.orphanEventCursor, - orphanEventsCompletedAt: state.orphanEventsCompletedAt, + eventCursor: state.eventCursor, + eventsCompletedAt: state.eventsCompletedAt, lastError: inputPersist.lastError, }) + return wrote + } + + if (await isDeleting()) { + return stopForDeletion() } // Baseline before scanning so mid-backfill updates stay inside the replay window. @@ -566,23 +315,33 @@ async function reconcileOneUser(input: { state, nowIso: input.nowIso, }) + if (await isDeleting()) { + return stopForDeletion() + } const messagePass = await backfillMessagesForUser({ env: input.env, state, deadlineMs: input.deadlineMs, + isDeleting, }) state = messagePass.state - initialBackfilled += messagePass.backfilled + creationBackfilled += messagePass.backfilled backfilled += messagePass.backfilled + if (messagePass.deletionStarted) { + return stopForDeletion() + } if (messagePass.retryableFailure) { - await persistIncomplete({ - lastError: 'message backfill mirror error or timeout', + await persistProgress({ + matchingSince: null, + lastError: messagePass.blockedReason + ? `message backfill ${messagePass.blockedReason}` + : 'message backfill mirror error or timeout', }) console.warn( 'mailbox-parity-message-backfill-retryable', state.userId, - 'error-or-timeout', + messagePass.blockedReason ?? 'error-or-timeout', ) return { backfilled, @@ -594,7 +353,7 @@ async function reconcileOneUser(input: { } } if (messagePass.budgetExhausted) { - await persistIncomplete({ lastError: null }) + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -605,22 +364,29 @@ async function reconcileOneUser(input: { } } - const orphanPass = await backfillOrphanEventsForUser({ + const eventPass = await backfillEventsForUser({ env: input.env, state, deadlineMs: input.deadlineMs, + isDeleting, }) - state = orphanPass.state - initialBackfilled += orphanPass.backfilled - backfilled += orphanPass.backfilled - if (orphanPass.retryableFailure) { - await persistIncomplete({ - lastError: 'orphan-event backfill mirror error or timeout', + state = eventPass.state + creationBackfilled += eventPass.backfilled + backfilled += eventPass.backfilled + if (eventPass.deletionStarted) { + return stopForDeletion() + } + if (eventPass.retryableFailure) { + await persistProgress({ + matchingSince: null, + lastError: eventPass.blockedReason + ? `event backfill ${eventPass.blockedReason}` + : 'event backfill mirror error or timeout', }) console.warn( - 'mailbox-parity-orphan-event-backfill-retryable', + 'mailbox-parity-event-backfill-retryable', state.userId, - 'error-or-timeout', + eventPass.blockedReason ?? 'error-or-timeout', ) return { backfilled, @@ -631,8 +397,8 @@ async function reconcileOneUser(input: { budgetExhausted: false, } } - if (orphanPass.budgetExhausted) { - await persistIncomplete({ lastError: null }) + if (eventPass.budgetExhausted) { + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -643,11 +409,8 @@ async function reconcileOneUser(input: { } } - if ( - state.messagesCompletedAt == null || - state.orphanEventsCompletedAt == null - ) { - await persistIncomplete({ lastError: null }) + if (state.messagesCompletedAt == null || state.eventsCompletedAt == null) { + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -658,9 +421,9 @@ async function reconcileOneUser(input: { } } - // Initial creation/orphan work ends the continuous soak window. Successful - // content replay below may keep matching_since when the final compare matches. - if (initialBackfilled > 0) { + // Creation work ends the continuous soak window. Successful content replay + // below may keep matching_since when the final compare matches. + if (creationBackfilled > 0) { state = { ...state, matchingSince: null } } @@ -669,17 +432,31 @@ async function reconcileOneUser(input: { state, nowIso: input.nowIso, deadlineMs: input.deadlineMs, + isDeleting, }) state = contentPass.state backfilled += contentPass.backfilled + if (contentPass.deletionStarted) { + return stopForDeletion() + } + // Durably freeze a newly opened window even when this tick cannot finish it. + if (contentPass.openedWindow && !contentPass.windowComplete) { + await persistProgress({ + matchingSince: null, + lastError: null, + }) + } if (contentPass.retryableFailure) { - await persistIncomplete({ - lastError: 'content replay mirror error or timeout', + await persistProgress({ + matchingSince: null, + lastError: contentPass.blockedReason + ? `content replay ${contentPass.blockedReason}` + : 'content replay mirror error or timeout', }) console.warn( 'mailbox-parity-content-replay-retryable', state.userId, - 'error-or-timeout', + contentPass.blockedReason ?? 'error-or-timeout', ) return { backfilled, @@ -691,7 +468,8 @@ async function reconcileOneUser(input: { } } if (!contentPass.windowComplete) { - await persistIncomplete({ lastError: null }) + // Freeze/retain upper + cursor; clear soak on incomplete budget ticks. + await persistProgress({ matchingSince: null, lastError: null }) return { backfilled, compared: false, @@ -702,23 +480,21 @@ async function reconcileOneUser(input: { } } + if (await isDeleting()) { + return stopForDeletion() + } + const compared = await compareParityForUser({ env: input.env, state, nowIso: input.nowIso, }) state = compared.state - await persistUserParityProgress({ - db: input.env.APP_DB, - userId: state.userId, - nowIso: input.nowIso, + if (await isDeleting()) { + return stopForDeletion() + } + await persistProgress({ matchingSince: state.matchingSince, - mismatchCount: state.mismatchCount, - contentWatermarkAt: state.contentWatermarkAt, - messageCursor: state.messageCursor, - messagesCompletedAt: state.messagesCompletedAt, - orphanEventCursor: state.orphanEventCursor, - orphanEventsCompletedAt: state.orphanEventsCompletedAt, lastError: null, }) return { diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts index 243d0b2086..36d40d50b0 100644 --- a/packages/worker/src/email/mailbox-reconcile.workers.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -3,6 +3,7 @@ import { expect, test, vi } from 'vitest' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { mailboxLiveMirrorMaxEvents } from './mailbox-live-mirror.ts' import { countD1MailboxParity, reconcileMailboxParity, @@ -37,6 +38,7 @@ async function seedMessageGraph(input: { threadId: string createdAt: string withAttachment?: boolean + eventCount?: number }) { await env.APP_DB.prepare( `INSERT INTO email_threads ( @@ -75,18 +77,22 @@ async function seedMessageGraph(input: { .bind(`att-${input.messageId}`, input.messageId, input.createdAt) .run() } - await env.APP_DB.prepare( - `INSERT INTO email_delivery_events ( - id, message_id, user_id, event_type, provider, detail_json, created_at - ) VALUES (?, ?, ?, 'sent', 'kody', '{}', ?)`, - ) - .bind( - `evt-${input.messageId}`, - input.messageId, - input.userId, - input.createdAt, + const eventCount = input.eventCount ?? 1 + const baseMs = Date.parse(input.createdAt) + for (let index = 0; index < eventCount; index += 1) { + await env.APP_DB.prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, ?, ?, 'sent', 'kody', '{}', ?)`, ) - .run() + .bind( + `evt-${input.messageId}-${String(index).padStart(3, '0')}`, + input.messageId, + input.userId, + new Date(baseMs + index * 1000).toISOString(), + ) + .run() + } } async function seedOrphanEvent(input: { @@ -110,10 +116,11 @@ async function readParityState(userId: string) { mailbox_parity_matching_since AS matchingSince, mailbox_parity_mismatch_count AS mismatchCount, mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, mailbox_parity_message_backfill_cursor_id AS messageCursorId, mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_orphan_event_backfill_cursor_id AS orphanCursorId, - mailbox_parity_orphan_event_backfill_completed_at AS orphanEventsCompletedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt, mailbox_parity_last_error AS lastError FROM users WHERE stable_user_id = ?`, @@ -124,10 +131,11 @@ async function readParityState(userId: string) { matchingSince: string | null mismatchCount: number contentWatermarkAt: string | null + contentReplayUpperAt: string | null messageCursorId: string | null messagesCompletedAt: string | null - orphanCursorId: string | null - orphanEventsCompletedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null lastError: string | null }>() } @@ -142,10 +150,8 @@ async function parkOther(userId: string) { .run() } -test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves soak', async () => { +test('reconcileMailboxParity mismatch full reset and soak', async () => { silenceIncidentalRuntimeWarnings() - // Keep wall time aligned with each tick's nowIso so same-run content upper - // bounds stay deterministic (without freezing RPC timeout clocks). vi.useFakeTimers({ shouldAdvanceTime: true }) await ensureUsersTestSchema({ db: env.APP_DB }) await ensureEmailTestSchema(env.APP_DB) @@ -191,16 +197,16 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves }) const mailbox = rpcFor(userId) - // Warm the DO so the first 1s-bounded mirror RPC is not cold-start timeout. await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) const firstNow = new Date('2026-08-01T10:00:00.000Z') vi.setSystemTime(firstNow) + // 2 messages + 2 bound events + 1 orphan await expect( reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), ).resolves.toEqual({ scanned: 1, - backfilled: 3, + backfilled: 5, compared: 1, matched: 1, mismatched: 0, @@ -216,20 +222,15 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves checkedAt: firstNow.toISOString(), matchingSince: firstNow.toISOString(), contentWatermarkAt: expect.any(String), + contentReplayUpperAt: null, mismatchCount: 0, messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), messageCursorId: 'parity-msg-2', - orphanCursorId: 'parity-orphan-1', + eventCursorId: 'parity-orphan-1', lastError: null, }) - if (!afterMatch?.contentWatermarkAt) { - throw new Error('Expected parity content watermark after match.') - } - expect(afterMatch.contentWatermarkAt >= firstNow.toISOString()).toBe(true) - // Live D1 row after completed cursors: compare mismatches, reopens backfill - // while retaining cursors so the next tick repairs only the new row. await seedMessageGraph({ userId, messageId: 'parity-msg-3', @@ -259,11 +260,11 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves expect(afterMismatch).toMatchObject({ matchingSince: null, mismatchCount: 1, - contentWatermarkAt: expect.any(String), messagesCompletedAt: null, - orphanEventsCompletedAt: null, - messageCursorId: 'parity-msg-2', - orphanCursorId: 'parity-orphan-1', + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + contentReplayUpperAt: null, checkedAt: mismatchNow.toISOString(), }) expect(afterMismatch?.contentWatermarkAt).not.toBeNull() @@ -276,11 +277,12 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves await parkOther(otherUserId) const rematchNow = new Date('2026-08-01T12:00:00.000Z') vi.setSystemTime(rematchNow) + // Full historical rescan: 3 messages + 3 bound events + 1 orphan await expect( reconcileMailboxParity({ env, now: rematchNow, batchSize: 1 }), ).resolves.toEqual({ scanned: 1, - backfilled: 1, + backfilled: 7, compared: 1, matched: 1, mismatched: 0, @@ -296,9 +298,8 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves expect(afterRematch).toMatchObject({ matchingSince: rematchNow.toISOString(), mismatchCount: 0, - contentWatermarkAt: expect.any(String), messagesCompletedAt: expect.any(String), - orphanEventsCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), messageCursorId: 'parity-msg-3', }) @@ -322,10 +323,68 @@ test('reconcileMailboxParity mismatch reopens backfill, resets count, preserves }) const afterSoak = await readParityState(userId) expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) - expect(afterSoak?.contentWatermarkAt).toEqual(expect.any(String)) expect(afterSoak?.checkedAt).toBe(soakNow.toISOString()) expect(afterSoak?.mismatchCount).toBe(0) } finally { vi.useRealTimers() } }, 30_000) + +test('reconcileMailboxParity eventually repairs >100 bound delivery events', async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-parity-reconcile-user-failed', + 'mailbox-parity-message-backfill-retryable', + 'mailbox-parity-event-backfill-retryable', + 'mailbox-parity-content-replay-retryable', + 'mailbox-live-mirror-events-truncated', + ]) + await ensureUsersTestSchema({ db: env.APP_DB }) + await ensureEmailTestSchema(env.APP_DB) + + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z'`, + ).run() + + const heavyUserId = await seedParityUser({ + email: `heavy-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + const heavyEventCount = mailboxLiveMirrorMaxEvents + 5 + await seedMessageGraph({ + userId: heavyUserId, + messageId: 'heavy-msg-1', + threadId: 'heavy-thread-1', + createdAt: '2026-07-02T12:00:00.000Z', + eventCount: heavyEventCount, + }) + const heavyMailbox = rpcFor(heavyUserId) + await heavyMailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + let matched = 0 + let backfilled = 0 + for (let tick = 0; tick < 30; tick += 1) { + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(heavyUserId) + .run() + const metrics = await reconcileMailboxParity({ + env, + now: new Date(Date.UTC(2026, 7, 3, 10, tick)), + batchSize: 1, + }) + backfilled += metrics.backfilled + matched += metrics.matched + if (metrics.matched > 0) break + } + expect(matched).toBeGreaterThan(0) + expect(backfilled).toBeGreaterThan(mailboxLiveMirrorMaxEvents) + expect(await heavyMailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId: heavyUserId }), + ) + expect( + (await countD1MailboxParity({ db: env.APP_DB, userId: heavyUserId })) + .deliveryEvents, + ).toBe(heavyEventCount) +}, 60_000) diff --git a/packages/worker/src/users-test-schema.ts b/packages/worker/src/users-test-schema.ts index fd279cb1da..e2ae33bfef 100644 --- a/packages/worker/src/users-test-schema.ts +++ b/packages/worker/src/users-test-schema.ts @@ -84,12 +84,15 @@ const alwaysAdditiveColumns: Record = { }, mailbox_parity_last_error: { create: 'TEXT' }, mailbox_parity_content_watermark_at: { create: 'TEXT' }, + mailbox_parity_content_replay_upper_at: { create: 'TEXT' }, + mailbox_parity_content_replay_cursor_updated_at: { create: 'TEXT' }, + mailbox_parity_content_replay_cursor_id: { create: 'TEXT' }, mailbox_parity_message_backfill_cursor_created_at: { create: 'TEXT' }, mailbox_parity_message_backfill_cursor_id: { create: 'TEXT' }, mailbox_parity_message_backfill_completed_at: { create: 'TEXT' }, - mailbox_parity_orphan_event_backfill_cursor_created_at: { create: 'TEXT' }, - mailbox_parity_orphan_event_backfill_cursor_id: { create: 'TEXT' }, - mailbox_parity_orphan_event_backfill_completed_at: { create: 'TEXT' }, + mailbox_parity_event_backfill_cursor_created_at: { create: 'TEXT' }, + mailbox_parity_event_backfill_cursor_id: { create: 'TEXT' }, + mailbox_parity_event_backfill_completed_at: { create: 'TEXT' }, } /** From 93aba3fedc6682564e3c23f90ce899282cdd2417 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:07:17 +0000 Subject: [PATCH 13/62] chore: refresh Mailbox parity migration hash Co-authored-by: Kent C. Dodds --- tools/migration-ledger.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 379b71d275..7dd7de2e83 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -514,7 +514,7 @@ }, { "filename": "0125-mailbox-parity-state.sql", - "sha256": "d5a99f57d8db18943b7184c1c2cc2002c5ef6c20584def71b9860e823b0456cc" + "sha256": "ac22677b803abd75b66ec9ba331c2cff6694390409bbb1719c15ff0b9799aa89" } ] } From 9dc335e73267462295ed221629fe04889003ccda Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:22:32 +0000 Subject: [PATCH 14/62] fix(email): rebuild Mailbox on parity mismatch Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 91 ++++--- .../architecture/feature-flags.md | 3 +- .../migrations/0125-mailbox-parity-state.sql | 8 +- .../worker/src/email/mailbox-parity-repo.ts | 14 + .../email/mailbox-read-cutover.node.test.ts | 4 +- .../worker/src/email/mailbox-read-cutover.ts | 7 +- .../src/email/mailbox-reconcile.node.test.ts | 253 +++++++++++++++++- .../worker/src/email/mailbox-reconcile.ts | 81 +++++- .../email/mailbox-reconcile.workers.test.ts | 121 ++++++++- .../scheduled/scheduled-lanes.node.test.ts | 35 +-- .../worker/src/scheduled/scheduled-lanes.ts | 9 +- tools/migration-ledger.json | 2 +- 12 files changed, 522 insertions(+), 106 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e747ae096d..6b0231359d 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -277,7 +277,7 @@ The schema is defined by migrations in `packages/worker/migrations/`: message and **all** owner delivery-event backfill cursors/completion timestamps on the user row. Indexed by `(mailbox_parity_checked_at, stable_user_id)` for oldest-first discovery in - the hourly `mailbox_parity` reconcile lane (see + the every-5-minute `mailbox_parity` reconcile lane (see [Mailbox](#durable-objects-mailbox)). The same migration adds keyset-friendly composites on `email_messages(user_id, created_at, id)`, `email_messages(user_id, updated_at, id)`, and @@ -395,10 +395,10 @@ delivery-event table because Wrangler's emulated dataset has no SQL API. with a separate row shape defined in `packages/worker/src/email/mailbox-parity-events.ts` (`recordMailboxParityEvent`). Live mirror helpers record one namespaced outcome -per attempted operation; the hourly `mailbox_parity` reconcile lane records one -outcome per count comparison (`compare_threads`, `compare_messages`, -`compare_attachments`, `compare_delivery_events`). These rows do not feed the -admin insights charts today: +per attempted operation; the `mailbox_parity` reconcile lane (every five-minute +cron tick) records one outcome per count comparison (`compare_threads`, +`compare_messages`, `compare_attachments`, `compare_delivery_events`). These +rows do not feed the admin insights charts today: - `index1` — stable user id (per-user isolation; `system:email` is excluded) - `blob1` — namespaced event type (`mailbox_mirror:` or @@ -809,19 +809,23 @@ mirrors are **not** wired in this slice. Scheduled parity reconcile **is** wired (see below); read cutover is prepared but not flipped. **Scheduled parity reconcile (`mailbox_parity`)** runs as its own queue-isolated -scheduled lane (`packages/worker/src/scheduled/scheduled-lanes.ts`, +scheduled lane on **every** five-minute Worker cron tick +(`packages/worker/src/scheduled/scheduled-lanes.ts`, `packages/worker/src/email/mailbox-reconcile.ts`, `packages/worker/src/email/mailbox-parity-phases.ts`, -`packages/worker/src/email/mailbox-parity-repo.ts`). The five-minute Worker cron -dispatches it only on the top of each UTC hour (`shouldRunMailboxParityCron`); -production uses the same per-lane queue isolation as reconcile, retention, and -sibling lanes so a slow parity pass cannot consume another lane's budget. +`packages/worker/src/email/mailbox-parity-repo.ts`). Production uses the same +per-lane queue isolation as reconcile, retention, and sibling lanes so a slow +parity pass cannot consume another lane's budget. User discovery is **D1-only** — no Mailbox Durable Object enumeration. Each tick -selects up to eight non-deleting mail owners (`mailboxParityUserBatchSize`) -ordered by oldest `mailbox_parity_checked_at`, excluding `system:email`, who -have at least one `email_messages` or `email_delivery_events` row. Per-user work -is bounded by a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`). +selects up to sixteen non-deleting owners (`mailboxParityUserBatchSize`) ordered +by oldest `mailbox_parity_checked_at`, excluding `system:email`. Owners with D1 +mail (`email_messages` or `email_delivery_events`) are always eligible; +**previously tracked** owners (any non-null parity column on `users`) remain +discoverable even when D1 mail is empty so DO-only leftovers after the last-row +delete can be purged and reconciled. Never-tracked empty users stay out. +Per-user work shares a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`) +across the batch. **Account-deletion races:** `loadUserParityState`, `persistUserParityProgress`, and `rotateCheckedAt` require `deleting_at IS NULL` (zero-row updates are @@ -865,19 +869,25 @@ D1↔Mailbox count parity for phase-3 soak tracking. Set on the first exact compare when null; preserved across successful content replay + exact compare. Cleared on: creation backfill work in the same tick (`creationBackfilled > 0`), incomplete/budget-exhausted ticks, mirror retryable failures, content-replay -failure/incomplete, compare mismatch (also increments consecutive -`mailbox_parity_mismatch_count`, **resets both message and delivery-event -creation cursors and completion markers** for a full historical rescan while -retaining the content watermark and clearing any in-flight replay window), and -compare-path errors (`rotateCheckedAt` also clears soak so a poison tick cannot -preserve a false window). Resets `mismatch_count` to `0` on exact match. +failure/incomplete, and compare-path errors (`rotateCheckedAt` also clears soak +so a poison tick cannot preserve a false window). Resets `mismatch_count` to `0` +on exact match. + +**Count mismatch rebuild:** on mismatch the lane first **must** observe a +successful metadata-only `Mailbox.purge()` (DO SQLite / alarm state; no R2), +then resets **all** parity cursors, completion markers, content watermark, and +in-flight replay window for a full D1-authoritative rebuild on subsequent ticks +(increments consecutive `mailbox_parity_mismatch_count` and clears soak). If +purge fails (timeout/error), rebuild state is **not** reset — the user stays +ineligible for soak, `last_error` records the failure, and the next ticks retry +purge before advancing cursors. **Phase 3 read cutover (prepared, not flipped):** `packages/worker/src/email/mailbox-read-cutover.ts` defines a default-off `mailbox-read-cutover` feature flag plus `getOwnerEmailMessageById` / `isMailboxReadCutoverEnabled`. The gate requires the flag **and** per-user parity soak: `mailbox_parity_matching_since` at least 24h old, -`mailbox_parity_checked_at` fresh within 2h, +`mailbox_parity_checked_at` fresh within 6h, `mailbox_parity_mismatch_count === 0`, exact `stable_user_id` match, and `deleting_at IS NULL` (deleting accounts fail closed to D1). D1 errors, missing parity state, and an off flag fail closed to D1 reads. No live reader calls this @@ -914,14 +924,14 @@ namespaced outcome telemetry (`missing` and `timeout` included). Message and batch event repair each use one 1s-bounded RPC; each graph attempt emits at most two AE writes; event repair batches up to 100 events with explicit truncation (newest retained, chronological insertion restored, stable truncation warning) -and one batch telemetry outcome. **`system:email` stays in D1 only.** The hourly -queue-isolated `mailbox_parity` lane backfills all owner messages and delivery -events, durable content-watermark replays, compares owner-scoped D1 vs Mailbox -counts, persists soak state on `users` (migration `0125`), and re-purges the DO -when account deletion races the lane. Inbound lifecycle dual-write, -retention/metadata deletes, and live read cutover are **still not wired**. Live -email read/write authority is unchanged — D1 `email_*` tables remain -authoritative for all live paths. D1 email rows and existing R2 inventory +and one batch telemetry outcome. **`system:email` stays in D1 only.** The +every-5-minute queue-isolated `mailbox_parity` lane backfills all owner messages +and delivery events, durable content-watermark replays, compares owner-scoped D1 +vs Mailbox counts, persists soak state on `users` (migration `0125`), and +re-purges the DO when account deletion races the lane. Inbound lifecycle +dual-write, retention/metadata deletes, and live read cutover are **still not +wired**. Live email read/write authority is unchanged — D1 `email_*` tables +remain authoritative for all live paths. D1 email rows and existing R2 inventory deletion stay authoritative during expand. **Phase 2 completes only when every user-mail mutation that writes D1 also writes the DO** — inbound paths and delete mirrors remain pending. @@ -946,8 +956,8 @@ delete mirrors remain pending. classification (`service.ts#setEmailMessageClassification`) call `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` outcomes automatically (graph repair uses message RPC + one batch event RPC; - at most two AE writes per attempt). The hourly `mailbox_parity` scheduled - lane backfills all owner messages and delivery events, durable + at most two AE writes per attempt). The every-5-minute `mailbox_parity` + scheduled lane backfills all owner messages and delivery events, durable content-watermark replays, count compares, and soak tracking on `users` (migration `0125`). **Still pending for phase-2 completion:** inbound commit/touch/classify dual-write and retention/metadata-delete mirrors. @@ -965,12 +975,13 @@ delete mirrors remain pending. 5. **Later contract migrations** — drop retired D1 user-mail tables/columns only after verification. No premature schema deletion. -The hourly `mailbox_parity` scheduled lane (queue-isolated sibling in +The every-5-minute `mailbox_parity` scheduled lane (queue-isolated sibling in `scheduled-lanes.ts`) owns backfill of all owner messages and delivery events, -durable content-watermark replay, count parity, soak state on `users`, and -best-effort `Mailbox.purge()` when account deletion races the lane. This Mailbox -track owns DO storage semantics, alarm retention, and the inbound durability -boundary below. +durable content-watermark replay, count parity, soak state on `users`, +metadata-only DO purge + full rebuild on count mismatch, and best-effort +`Mailbox.purge()` when account deletion races the lane. This Mailbox track owns +DO storage semantics, alarm retention, and the inbound durability boundary +below. ### What stays in D1 @@ -1263,9 +1274,11 @@ Artifacts tokens. Reconcile runs through the registry in `packages/worker/src/scheduled/scheduled-lanes.ts`, alongside repo-session cleanup, system-email retention, general retention, job retention, hourly -usage-rollup aggregation, and the hourly `mailbox_parity` lane (top of each UTC -hour only; D1 user discovery, bounded message + full delivery-event backfill, -durable content-watermark replay, and count compare — see +usage-rollup aggregation, and the every-5-minute `mailbox_parity` lane +(queue-isolated; up to sixteen users per tick within a ~10s budget; D1 user +discovery including previously tracked empty-mail owners, bounded message + full +delivery-event backfill, durable content-watermark replay, metadata-only purge + +full rebuild on count mismatch, and count compare — see [Mailbox](#durable-objects-mailbox)). Each production queue message preserves `scheduled_lane_failed` / D1 lock-contention log and Sentry context. A handled lane failure is acknowledged and retried by the next cron tick, matching the old diff --git a/docs/contributing/architecture/feature-flags.md b/docs/contributing/architecture/feature-flags.md index b026c1f661..f5e0b8a776 100644 --- a/docs/contributing/architecture/feature-flags.md +++ b/docs/contributing/architecture/feature-flags.md @@ -81,7 +81,8 @@ parity soak on `users` (migration `0125-mailbox-parity-state.sql`): - `mailbox_parity_matching_since` at least **24 hours** old (continuous exact D1↔Mailbox count parity), -- `mailbox_parity_checked_at` fresh within **2 hours** (hourly lane + slack), +- `mailbox_parity_checked_at` fresh within **6 hours** (every-5m lane with + bounded batch scaling; not a strict 5m SLA), - `mailbox_parity_mismatch_count === 0`, - exact `stable_user_id` match between the gate caller and the loaded row, - `deleting_at IS NULL` (accounts marked for deletion fail closed to D1). diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql index 55a93e2459..937a5a76e2 100644 --- a/packages/worker/migrations/0125-mailbox-parity-state.sql +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -9,9 +9,11 @@ -- content-replay window (replay_upper_at + replay cursor) keyset-replays -- messages with updated_at in (watermark, upper]; the watermark advances to -- upper only when that window finishes. Count compare runs only after the full --- event phase and a completed content window. Count mismatch clears soak --- eligibility and resets both creation cursors and completion markers for a --- full historical rescan (watermark preserved; in-flight replay window cleared). +-- event phase and a completed content window. Count mismatch best-effort purges +-- Mailbox SQLite metadata (no R2), then clears soak and resets creation cursors, +-- completion markers, and content watermark/window for an authoritative D1 +-- rebuild. Purge timeout/error clears soak, records failure, and does not reset +-- rebuild state. Previously tracked users stay eligible even at zero D1 rows. -- matching_since may persist across a successful content replay + exact compare. ALTER TABLE users ADD COLUMN mailbox_parity_checked_at TEXT; diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts index 6c9c97ccf1..2ee1a86e06 100644 --- a/packages/worker/src/email/mailbox-parity-repo.ts +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -45,6 +45,9 @@ export async function listUsersForMailboxParity(input: { db: D1Database limit: number }): Promise> { + // Include previously tracked owners (any parity column non-null) even when + // D1 mail is empty so DO-only leftovers after last-row deletion are purged. + // Never-tracked empty users, system email, and deleting users stay out. const result = await input.db .prepare( `SELECT u.stable_user_id AS userId @@ -60,6 +63,17 @@ export async function listUsersForMailboxParity(input: { SELECT 1 FROM email_delivery_events e WHERE e.user_id = u.stable_user_id ) + OR u.mailbox_parity_checked_at IS NOT NULL + OR u.mailbox_parity_matching_since IS NOT NULL + OR u.mailbox_parity_last_error IS NOT NULL + OR u.mailbox_parity_mismatch_count > 0 + OR u.mailbox_parity_content_watermark_at IS NOT NULL + OR u.mailbox_parity_content_replay_upper_at IS NOT NULL + OR u.mailbox_parity_content_replay_cursor_id IS NOT NULL + OR u.mailbox_parity_message_backfill_cursor_id IS NOT NULL + OR u.mailbox_parity_message_backfill_completed_at IS NOT NULL + OR u.mailbox_parity_event_backfill_cursor_id IS NOT NULL + OR u.mailbox_parity_event_backfill_completed_at IS NOT NULL ) ORDER BY u.mailbox_parity_checked_at ASC, u.stable_user_id ASC LIMIT ?`, diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts index 5888ef3d66..ad36092808 100644 --- a/packages/worker/src/email/mailbox-read-cutover.node.test.ts +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -272,7 +272,7 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch }), ).resolves.toBe(false) - // Stale checked_at (> 2h). + // Stale checked_at (> 6h freshness window). const staleDb = createCutoverTestDb({ flags: { overrideEnabled: true, globalEnabled: null }, parityByUserId: new Map([ @@ -280,7 +280,7 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch dbUserId, { ...soakedParity(), - mailbox_parity_checked_at: hoursAgoIso(3), + mailbox_parity_checked_at: hoursAgoIso(7), }, ], ]), diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts index c6058323b8..40ad9fbb81 100644 --- a/packages/worker/src/email/mailbox-read-cutover.ts +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -27,8 +27,11 @@ import { type EmailMessageRecord } from './types.ts' /** Continuous exact parity must hold at least this long before cutover. */ export const mailboxReadCutoverSoakMs = 24 * 60 * 60 * 1000 -/** Parity `checked_at` must be newer than this (hourly lane + slack). */ -export const mailboxReadCutoverCheckedAtMaxAgeMs = 2 * 60 * 60 * 1000 +/** + * Parity `checked_at` must be newer than this. Sized for the every-5m parity + * lane with bounded batch scaling (not a strict 5m SLA). + */ +export const mailboxReadCutoverCheckedAtMaxAgeMs = 6 * 60 * 60 * 1000 export const mailboxReadCutoverFlagKey = 'mailbox-read-cutover' as const diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts index 8b639c4787..f082b68e95 100644 --- a/packages/worker/src/email/mailbox-reconcile.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.node.test.ts @@ -185,20 +185,27 @@ function successGraph(messageId: string, eventsTruncated = false) { } } -function mockDoCounts(counts: { - threads: number - messages: number - attachments: number - deliveryEvents: number -}) { - mocks.awaitMailboxMirrorRpc.mockResolvedValue({ - ok: true, - value: counts, - }) +function mockDoCounts( + counts: { + threads: number + messages: number + attachments: number + deliveryEvents: number + }, + options?: { purge?: () => Promise<{ ok: true }> }, +) { + const purge = options?.purge ?? vi.fn(async () => ({ ok: true as const })) mocks.mailboxRpc.mockImplementation(() => ({ countMailbox: vi.fn(async () => counts), - purge: vi.fn(async () => ({ ok: true as const })), + purge, })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => ({ + ok: true as const, + value: await promise, + }), + ) + return { purge } } function parityEnv(db: D1Database) { @@ -416,7 +423,7 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou checkedAt: mismatchNow.toISOString(), matchingSince: null, mismatchCount: 3, - contentWatermarkAt: mismatchNow.toISOString(), + contentWatermarkAt: null, contentReplayUpperAt: null, contentReplayCursorId: null, messagesCompletedAt: null, @@ -463,12 +470,16 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou expect(afterRematch).toMatchObject({ matchingSince: rematchNow.toISOString(), mismatchCount: 0, - contentWatermarkAt: rematchNow.toISOString(), + contentWatermarkAt: expect.any(String), messagesCompletedAt: expect.any(String), eventsCompletedAt: expect.any(String), messageCursorId: 'msg-live-new', eventCursorId: 'evt-orphan-1', }) + if (!afterRematch?.contentWatermarkAt) { + throw new Error('Expected content watermark after parity rematch.') + } + expect(afterRematch.contentWatermarkAt >= rematchNow.toISOString()).toBe(true) await parkUser(db, ownerB) const soakNow = new Date('2026-08-02T12:00:00.000Z') @@ -997,3 +1008,219 @@ test('mailbox parity deletion race purges Mailbox and skips parity writes', asyn expect(row?.matchingSince).toBeNull() expect(row?.eventsCompletedAt).toBeNull() }) + +test('tracked empty user discovers DO-only leftover, purges, then reaches zero parity', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'k'.repeat(64) + const neverTracked = 'l'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertUser({ db, userId: neverTracked, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + + // Simulate failed delete mirror: D1 cleared, DO still holds message/event. + await db + .prepare(`DELETE FROM email_delivery_events WHERE user_id = ?`) + .bind(owner) + .run() + await db + .prepare(`DELETE FROM email_messages WHERE user_id = ?`) + .bind(owner) + .run() + + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).toEqual([owner]) + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).not.toContain(neverTracked) + + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const afterPurge = await readParityRow(db, owner) + expect(afterPurge).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + lastError: null, + }) + + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + const zeroNow = new Date('2026-08-01T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterZero = await readParityRow(db, owner) + expect(afterZero).toMatchObject({ + matchingSince: zeroNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + }) + if (!afterZero?.contentWatermarkAt) { + throw new Error('Expected content watermark after zero-count parity.') + } + expect(afterZero.contentWatermarkAt >= zeroNow.toISOString()).toBe(true) +}) + +test('mismatch purge failure clears soak and does not start rebuild reset', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'm'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1 }) + const afterMatch = await readParityRow(db, owner) + expect(afterMatch?.messagesCompletedAt).toEqual(expect.any(String)) + expect(afterMatch?.contentWatermarkAt).toBe(now.toISOString()) + + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + })), + purge: vi.fn(async () => ({ ok: true as const })), + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + const settled = await promise + if (settled && typeof settled === 'object' && 'messages' in settled) { + return { ok: true as const, value: settled } + } + return { ok: false as const, timedOut: true as const } + }, + ) + consoleWarn.mockClear() + const failNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(failNow) + await expect( + reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-mismatch-purge-failed', + owner, + 'timeout', + 'mailbox metadata purge timeout', + ) + const afterFail = await readParityRow(db, owner) + // Empty content window may advance the watermark before compare; purge + // failure must not clear creation completion/cursors (rebuild not started). + expect(afterFail).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + messagesCompletedAt: afterMatch?.messagesCompletedAt, + eventsCompletedAt: afterMatch?.eventsCompletedAt, + messageCursorId: afterMatch?.messageCursorId, + lastError: 'mailbox metadata purge timeout', + checkedAt: failNow.toISOString(), + }) + expect(afterFail?.contentWatermarkAt).not.toBeNull() + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index d2c9943930..c1cea9693b 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -41,7 +41,7 @@ import { type MailboxCountResult } from './mailbox-types.ts' * lane never mutates email_* rows and does not flip read authority. */ -export const mailboxParityUserBatchSize = 8 +export const mailboxParityUserBatchSize = 16 export const mailboxParityTimeBudgetMs = 10_000 export const mailboxParityLastErrorMaxBytes = 512 @@ -85,17 +85,49 @@ function safeParityErrorText(error: unknown): string { ) } -async function purgeMailboxBestEffort(input: { +/** + * Bounded Mailbox SQLite metadata purge (no R2). Used for account-deletion + * races (best-effort) and count-mismatch rebuild (must observe success). + */ +async function purgeMailboxMetadata(input: { env: MailboxParityReconcileEnv userId: string -}) { +}): Promise< + { ok: true } | { ok: false; reason: 'timeout' | 'error'; errorText: string } +> { try { - await awaitMailboxMirrorRpc( + const raced = await awaitMailboxMirrorRpc( mailboxRpc({ env: input.env, userId: input.userId }).purge(), mailboxMirrorRpcTimeoutMs, ) + if (!raced.ok) { + return { + ok: false, + reason: 'timeout', + errorText: 'mailbox metadata purge timeout', + } + } + return { ok: true } } catch (error) { - console.warn('mailbox-parity-purge-on-delete-failed', input.userId, error) + return { + ok: false, + reason: 'error', + errorText: safeParityErrorText(error), + } + } +} + +async function purgeMailboxBestEffort(input: { + env: MailboxParityReconcileEnv + userId: string +}) { + const result = await purgeMailboxMetadata(input) + if (!result.ok) { + console.warn( + 'mailbox-parity-purge-on-delete-failed', + input.userId, + result.errorText, + ) } } @@ -185,6 +217,8 @@ async function compareParityForUser(input: { }): Promise<{ state: MailboxParityUserState matched: boolean + purgeFailed: boolean + lastError: string | null }> { const d1 = await countD1MailboxParity({ db: input.env.APP_DB, @@ -213,10 +247,36 @@ async function compareParityForUser(input: { mismatchCount: 0, }, matched: true, + purgeFailed: false, + lastError: null, } } - // Full historical rescan: clear creation cursors + completion markers. - // Preserve content watermark; drop any in-flight replay window; clear soak. + + // DO may hold extras after D1 deletes. Purge Mailbox metadata (no R2) before + // resetting rebuild state so the next ticks authoritatively mirror D1. + const purged = await purgeMailboxMetadata({ + env: input.env, + userId: input.state.userId, + }) + if (!purged.ok) { + console.warn( + 'mailbox-parity-mismatch-purge-failed', + input.state.userId, + purged.reason, + purged.errorText, + ) + return { + state: { + ...input.state, + matchingSince: null, + mismatchCount: input.state.mismatchCount + 1, + }, + matched: false, + purgeFailed: true, + lastError: purged.errorText, + } + } + return { state: { ...input.state, @@ -226,10 +286,13 @@ async function compareParityForUser(input: { messagesCompletedAt: null, eventCursor: null, eventsCompletedAt: null, + contentWatermarkAt: null, contentReplayUpperAt: null, contentReplayCursor: null, }, matched: false, + purgeFailed: false, + lastError: null, } } @@ -495,14 +558,14 @@ async function reconcileOneUser(input: { } await persistProgress({ matchingSince: state.matchingSince, - lastError: null, + lastError: compared.lastError, }) return { backfilled, compared: true, matched: compared.matched, mismatched: !compared.matched, - failed: false, + failed: compared.purgeFailed, budgetExhausted: false, } } diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts index 36d40d50b0..fd0350d209 100644 --- a/packages/worker/src/email/mailbox-reconcile.workers.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -6,9 +6,14 @@ import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' import { mailboxLiveMirrorMaxEvents } from './mailbox-live-mirror.ts' import { countD1MailboxParity, + listUsersForMailboxParity, reconcileMailboxParity, } from './mailbox-reconcile.ts' -import { rpcFor } from './mailbox-test-helpers.ts' +import { + baseDeliveryEvent, + baseMessage, + rpcFor, +} from './mailbox-test-helpers.ts' import { ensureEmailTestSchema } from './test-schema.ts' async function seedParityUser(input: { @@ -264,10 +269,10 @@ test('reconcileMailboxParity mismatch full reset and soak', async () => { eventsCompletedAt: null, messageCursorId: null, eventCursorId: null, + contentWatermarkAt: null, contentReplayUpperAt: null, checkedAt: mismatchNow.toISOString(), }) - expect(afterMismatch?.contentWatermarkAt).not.toBeNull() await env.APP_DB.prepare( `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, @@ -330,6 +335,118 @@ test('reconcileMailboxParity mismatch full reset and soak', async () => { } }, 30_000) +test('reconcileMailboxParity purges DO-only leftover for tracked empty owners', async () => { + silenceIncidentalRuntimeWarnings(['mailbox-live-mirror-events-truncated']) + await ensureUsersTestSchema({ db: env.APP_DB }) + await ensureEmailTestSchema(env.APP_DB) + + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z'`, + ).run() + + const userId = await seedParityUser({ + email: `do-extra-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + await seedMessageGraph({ + userId, + messageId: 'do-extra-msg', + threadId: 'do-extra-thread', + createdAt: '2026-07-01T12:00:00.000Z', + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const firstNow = new Date('2026-08-01T10:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1 }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) + + // Failed delete mirror: clear D1, leave Mailbox metadata populated. + await env.APP_DB.prepare( + `DELETE FROM email_delivery_events WHERE user_id = ?`, + ) + .bind(userId) + .run() + await env.APP_DB.prepare(`DELETE FROM email_attachments WHERE message_id = ?`) + .bind('do-extra-msg') + .run() + await env.APP_DB.prepare(`DELETE FROM email_messages WHERE user_id = ?`) + .bind(userId) + .run() + await env.APP_DB.prepare(`DELETE FROM email_threads WHERE user_id = ?`) + .bind(userId) + .run() + await mailbox.mirrorMessage({ + ownerId: userId, + message: baseMessage(userId, { id: 'do-extra-msg' }), + }) + await mailbox.upsertDeliveryEvents({ + ownerId: userId, + events: [ + baseDeliveryEvent({ + id: 'do-extra-evt', + messageId: 'do-extra-msg', + }), + ], + }) + expect((await mailbox.countMailbox()).messages).toBeGreaterThan(0) + expect( + (await listUsersForMailboxParity({ db: env.APP_DB, limit: 10 })).some( + (row) => row.userId === userId, + ), + ).toBe(true) + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(await readParityState(userId)).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + }) + expect(await mailbox.countMailbox()).toEqual({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + + await env.APP_DB.prepare( + `UPDATE users SET mailbox_parity_checked_at = NULL WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + const zeroNow = new Date('2026-08-01T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, mismatched: 0 }) + expect(await readParityState(userId)).toMatchObject({ + matchingSince: zeroNow.toISOString(), + mismatchCount: 0, + }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) +}, 30_000) + test('reconcileMailboxParity eventually repairs >100 bound delivery events', async () => { silenceIncidentalRuntimeWarnings([ 'mailbox-parity-reconcile-user-failed', diff --git a/packages/worker/src/scheduled/scheduled-lanes.node.test.ts b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts index dcd75153d4..9bb5eacf74 100644 --- a/packages/worker/src/scheduled/scheduled-lanes.node.test.ts +++ b/packages/worker/src/scheduled/scheduled-lanes.node.test.ts @@ -8,36 +8,19 @@ vi.mock('#worker/email/mailbox-reconcile.ts', () => ({ reconcileMailboxParity: mocks.reconcileMailboxParity, })) -const { getScheduledLanes, runScheduledLane, shouldRunMailboxParityCron } = +const { getScheduledLanes, runScheduledLane } = await import('./scheduled-lanes.ts') -test('shouldRunMailboxParityCron gates to the top of each UTC hour', () => { - expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:00:30.000Z'))).toBe( - true, - ) - expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:05:00.000Z'))).toBe( - false, - ) - expect(shouldRunMailboxParityCron(new Date('2026-07-05T10:30:00.000Z'))).toBe( - false, - ) -}) - -test('getScheduledLanes includes mailbox_parity only on the hourly gate', () => { +test('getScheduledLanes includes mailbox_parity on every cron tick', () => { const env = {} as Env - expect( - getScheduledLanes({ - env, - scheduledAt: new Date('2026-07-05T10:00:30.000Z'), - }), - ).toContain('mailbox_parity') - expect( - getScheduledLanes({ - env, - scheduledAt: new Date('2026-07-05T10:30:00.000Z'), - }), - ).not.toContain('mailbox_parity') + for (const scheduledAt of [ + new Date('2026-07-05T10:00:30.000Z'), + new Date('2026-07-05T10:05:00.000Z'), + new Date('2026-07-05T10:30:00.000Z'), + ]) { + expect(getScheduledLanes({ env, scheduledAt })).toContain('mailbox_parity') + } }) test('runScheduledLane dispatches mailbox_parity through reconcileMailboxParity', async () => { diff --git a/packages/worker/src/scheduled/scheduled-lanes.ts b/packages/worker/src/scheduled/scheduled-lanes.ts index 1ff5b09dee..34a143f539 100644 --- a/packages/worker/src/scheduled/scheduled-lanes.ts +++ b/packages/worker/src/scheduled/scheduled-lanes.ts @@ -33,11 +33,6 @@ import { shouldRunUsageAggregationCron, } from '#worker/usage/aggregate-rollups.ts' -/** Worker cron fires every 5 minutes; run this lane on the top of each UTC hour. */ -export function shouldRunMailboxParityCron(now: Date) { - return now.getUTCMinutes() === 0 -} - export const scheduledLaneNames = [ 'reconcile_artifacts_pushes', 'repo_session_cleanup', @@ -86,6 +81,7 @@ export function getScheduledLanes(input: { 'storage_bucket_estimate_backfill', 'd1_storage_reconciliation', 'oauth_purge_expired', + 'mailbox_parity', ] if (shouldRunRetentionCron(input.scheduledAt)) { lanes.push('retention', 'job_retention') @@ -114,9 +110,6 @@ export function getScheduledLanes(input: { if (shouldRunJobScheduleWatchdogCron(input.scheduledAt)) { lanes.push('job_schedule_watchdog') } - if (shouldRunMailboxParityCron(input.scheduledAt)) { - lanes.push('mailbox_parity') - } return lanes } diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 7dd7de2e83..7cd41d25a2 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -514,7 +514,7 @@ }, { "filename": "0125-mailbox-parity-state.sql", - "sha256": "ac22677b803abd75b66ec9ba331c2cff6694390409bbb1719c15ff0b9799aa89" + "sha256": "a0cfd916aff30eecf74d315088c131a8fd9f800f44aa8ed4406a806cfa1f1cff" } ] } From 48d23b7833b94522cd62a6f553ade0762b21e446 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:23:37 +0000 Subject: [PATCH 15/62] test(email): isolate Mailbox parity lane Co-authored-by: Kent C. Dodds --- packages/worker/src/index.workers.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/worker/src/index.workers.test.ts b/packages/worker/src/index.workers.test.ts index 460f1f15bb..701688a659 100644 --- a/packages/worker/src/index.workers.test.ts +++ b/packages/worker/src/index.workers.test.ts @@ -57,6 +57,14 @@ const mocks = vi.hoisted(() => ({ updated: 0, failed: 0, })), + reconcileMailboxParity: vi.fn(async () => ({ + scanned: 0, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + })), })) vi.mock('./jobs/reconcile-artifacts-pushes.ts', () => ({ @@ -75,6 +83,10 @@ vi.mock('#worker/email/reconcile-inbound-deliveries.ts', () => ({ sweepStaleInboundDeliveries: mocks.sweepStaleInboundDeliveries, })) +vi.mock('#worker/email/mailbox-reconcile.ts', () => ({ + reconcileMailboxParity: mocks.reconcileMailboxParity, +})) + vi.mock('#app/retention.ts', () => ({ pruneRetention: mocks.pruneRetention, shouldRunRetentionCron: mocks.shouldRunRetentionCron, From 4c73520485c9bd8f1b531a8348634a6d91ba1a85 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:35:25 +0000 Subject: [PATCH 16/62] refactor(email): clarify Mailbox parity phases Co-authored-by: Kent C. Dodds --- .../worker/src/email/mailbox-parity-phases.ts | 208 ++---- ...ncile-cursor-backfill-content.node.test.ts | 626 ++++++++++++++++++ ...ncile-mismatch-deletion-error.node.test.ts | 423 ++++++++++++ .../email/mailbox-reconcile-test-support.ts | 219 ++++++ .../worker/src/email/mailbox-reconcile.ts | 476 ++++++++----- 5 files changed, 1627 insertions(+), 325 deletions(-) create mode 100644 packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts create mode 100644 packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts create mode 100644 packages/worker/src/email/mailbox-reconcile-test-support.ts diff --git a/packages/worker/src/email/mailbox-parity-phases.ts b/packages/worker/src/email/mailbox-parity-phases.ts index 6a5306f98d..a7daa3b7dd 100644 --- a/packages/worker/src/email/mailbox-parity-phases.ts +++ b/packages/worker/src/email/mailbox-parity-phases.ts @@ -21,15 +21,41 @@ export type MailboxParityPhaseEnv = MailboxLiveMirrorEnv & { APP_DB: D1Database } -export type MailboxParityPhaseResult = { +type MailboxParityPhaseBase = { state: MailboxParityUserState backfilled: number - budgetExhausted: boolean - retryableFailure: boolean - /** Set when a selected mirror outcome must not advance the cursor. */ - blockedReason: string | null } +/** Creation (message/event) phase outcome. */ +export type MailboxParityCreationPhaseResult = + | (MailboxParityPhaseBase & { status: 'complete' }) + | (MailboxParityPhaseBase & { status: 'budget_exhausted' }) + | (MailboxParityPhaseBase & { + status: 'retryable_failure' + blockedReason: string + }) + | (MailboxParityPhaseBase & { status: 'deletion_started' }) + +/** Content-replay phase outcome (`complete` ≡ full window finished). */ +export type MailboxParityContentPhaseResult = + | (MailboxParityPhaseBase & { + status: 'complete' + openedWindow: boolean + }) + | (MailboxParityPhaseBase & { + status: 'budget_exhausted' + openedWindow: boolean + }) + | (MailboxParityPhaseBase & { + status: 'retryable_failure' + blockedReason: string + openedWindow: boolean + }) + | (MailboxParityPhaseBase & { + status: 'deletion_started' + openedWindow: boolean + }) + /** * Message graph cursor may advance only on mirrored/stale. missing / skipped / * timeout / error must not advance (selected D1 row was expected). @@ -132,18 +158,11 @@ export async function backfillMessagesForUser(input: { state: MailboxParityUserState deadlineMs: number isDeleting: () => Promise -}): Promise { +}): Promise { let state = input.state let backfilled = 0 if (state.messagesCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } while (Date.now() < input.deadlineMs) { @@ -158,39 +177,18 @@ export async function backfillMessagesForUser(input: { ...state, messagesCompletedAt: new Date().toISOString(), } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } let cursor = state.messageCursor for (const row of page) { if (Date.now() >= input.deadlineMs) { state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } if (await input.isDeleting()) { state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } const summary = await mirrorMailboxMessageGraphFromD1({ env: input.env, @@ -200,24 +198,15 @@ export async function backfillMessagesForUser(input: { }) if (await input.isDeleting()) { state = { ...state, messageCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } if (!messageGraphAllowsCursorAdvance(summary)) { state = { ...state, messageCursor: cursor } return { + status: 'retryable_failure', state, backfilled, - budgetExhausted: false, - retryableFailure: true, blockedReason: `message graph ${summary.message.status}`, - deletionStarted: false, } } cursor = { createdAt: row.created_at, id: row.id } @@ -226,14 +215,7 @@ export async function backfillMessagesForUser(input: { state = { ...state, messageCursor: cursor } } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } export async function backfillEventsForUser(input: { @@ -241,28 +223,14 @@ export async function backfillEventsForUser(input: { state: MailboxParityUserState deadlineMs: number isDeleting: () => Promise -}): Promise { +}): Promise { let state = input.state let backfilled = 0 if (state.messagesCompletedAt == null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } if (state.eventsCompletedAt != null) { - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } while (Date.now() < input.deadlineMs) { @@ -277,39 +245,18 @@ export async function backfillEventsForUser(input: { ...state, eventsCompletedAt: new Date().toISOString(), } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'complete', state, backfilled } } let cursor = state.eventCursor for (const row of page) { if (Date.now() >= input.deadlineMs) { state = { ...state, eventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } if (await input.isDeleting()) { state = { ...state, eventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } const result = await mirrorMailboxParityDeliveryEventFromD1({ env: input.env, @@ -319,27 +266,18 @@ export async function backfillEventsForUser(input: { }) if (await input.isDeleting()) { state = { ...state, eventCursor: cursor } - return { - state, - backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - } + return { status: 'deletion_started', state, backfilled } } if (!eventMirrorAllowsCursorAdvance(result)) { state = { ...state, eventCursor: cursor } return { + status: 'retryable_failure', state, backfilled, - budgetExhausted: false, - retryableFailure: true, blockedReason: result.status === 'skipped' ? `event skipped:${result.reason}` : `event ${result.status}`, - deletionStarted: false, } } cursor = { createdAt: row.created_at, id: row.id } @@ -348,14 +286,7 @@ export async function backfillEventsForUser(input: { state = { ...state, eventCursor: cursor } } - return { - state, - backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - } + return { status: 'budget_exhausted', state, backfilled } } /** @@ -368,13 +299,7 @@ export async function replayContentUpdatesForUser(input: { nowIso: string deadlineMs: number isDeleting: () => Promise -}): Promise< - MailboxParityPhaseResult & { - deletionStarted: boolean - windowComplete: boolean - openedWindow: boolean - } -> { +}): Promise { let state = input.state const watermarkAt = state.contentWatermarkAt ?? input.nowIso let openedWindow = false @@ -408,6 +333,7 @@ export async function replayContentUpdatesForUser(input: { }) if (page.length === 0) { return { + status: 'complete', state: { ...state, contentWatermarkAt: upperBoundAt, @@ -415,11 +341,6 @@ export async function replayContentUpdatesForUser(input: { contentReplayCursor: null, }, backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - windowComplete: true, openedWindow, } } @@ -427,33 +348,25 @@ export async function replayContentUpdatesForUser(input: { for (const row of page) { if (Date.now() >= input.deadlineMs) { return { + status: 'budget_exhausted', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - windowComplete: false, openedWindow, } } if (await input.isDeleting()) { return { + status: 'deletion_started', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - windowComplete: false, openedWindow, } } @@ -465,33 +378,26 @@ export async function replayContentUpdatesForUser(input: { }) if (await input.isDeleting()) { return { + status: 'deletion_started', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: false, - retryableFailure: false, - blockedReason: null, - deletionStarted: true, - windowComplete: false, openedWindow, } } if (!messageGraphAllowsCursorAdvance(summary)) { return { + status: 'retryable_failure', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: false, - retryableFailure: true, blockedReason: `content replay ${summary.message.status}`, - deletionStarted: false, - windowComplete: false, openedWindow, } } @@ -506,17 +412,13 @@ export async function replayContentUpdatesForUser(input: { } return { + status: 'budget_exhausted', state: { ...state, contentReplayUpperAt: upperBoundAt, contentReplayCursor: cursor, }, backfilled, - budgetExhausted: true, - retryableFailure: false, - blockedReason: null, - deletionStarted: false, - windowComplete: false, openedWindow, } } diff --git a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts new file mode 100644 index 0000000000..9e509c5547 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts @@ -0,0 +1,626 @@ +import { + countD1MailboxParity, + createParityDb, + getMailboxReconcileTestMocks, + insertEvent, + insertMessage, + insertUser, + listEventBackfillPage, + listUsersForMailboxParity, + mockDoCounts, + parkUser, + parityEnv, + readParityRow, + reconcileMailboxParity, + successGraph, +} from './mailbox-reconcile-test-support.ts' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import { + mailboxParityContentPageSize, + mailboxParityMessagePageSize, +} from './mailbox-parity-phases.ts' + +const mocks = getMailboxReconcileTestMocks() + +test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const ownerA = 'a'.repeat(64) + const ownerB = 'b'.repeat(64) + const deleting = 'c'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + + await insertUser({ db, userId: ownerA, checkedAt: null }) + await insertUser({ + db, + userId: ownerB, + checkedAt: '2026-07-01T00:00:00.000Z', + }) + await insertUser({ + db, + userId: deleting, + deletingAt: '2026-07-31T00:00:00.000Z', + checkedAt: null, + }) + + const messageIds = Array.from( + { length: mailboxParityMessagePageSize + 1 }, + (_, index) => `msg-${String(index + 1).padStart(2, '0')}`, + ) + for (const [index, id] of messageIds.entries()) { + await insertMessage({ + db, + userId: ownerA, + id, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + await insertEvent({ + db, + userId: ownerA, + id: 'evt-bound-1', + messageId: messageIds[0]!, + createdAt: '2026-07-02T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: ownerA, + id: 'evt-orphan-1', + createdAt: '2026-07-02T00:00:01.000Z', + }) + await insertMessage({ + db, + userId: ownerB, + id: 'b-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: systemEmailOwnerId, + id: 'sys-msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: deleting, + id: 'del-evt', + createdAt: '2026-07-01T00:00:00.000Z', + }) + + const discovered = await listUsersForMailboxParity({ db, limit: 10 }) + expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) + expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) + expect(discovered.map((row) => row.userId)).not.toContain(deleting) + + const eventPage = await listEventBackfillPage({ + db, + userId: ownerA, + cursor: null, + limit: 10, + }) + expect(eventPage.map((row) => row.id)).toEqual([ + 'evt-bound-1', + 'evt-orphan-1', + ]) + + await db + .prepare( + `INSERT INTO email_threads ( + id, user_id, subject_normalized, last_message_at, created_at, updated_at + ) VALUES ('thread-a', ?, 'hi', ?, ?, ?)`, + ) + .bind( + ownerA, + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + '2026-07-01T00:00:00.000Z', + ) + .run() + + await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ + threads: 1, + messages: mailboxParityMessagePageSize + 1, + attachments: 0, + deliveryEvents: 2, + }) + + let graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + graphCalls += 1 + if (graphCalls === mailboxParityMessagePageSize) { + return { + messageId: input.messageId, + message: { status: 'timeout' as const }, + events: [], + eventsTruncated: false, + } + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + + const env = parityEnv(db) + await db + .prepare( + `UPDATE users + SET mailbox_parity_matching_since = '2026-07-01T00:00:00.000Z', + mailbox_parity_mismatch_count = 2 + WHERE stable_user_id = ?`, + ) + .bind(ownerA) + .run() + + await parkUser(db, ownerB) + const first = await reconcileMailboxParity({ + env, + now, + batchSize: 1, + }) + expect(first).toEqual({ + scanned: 1, + backfilled: mailboxParityMessagePageSize - 1, + compared: 0, + matched: 0, + mismatched: 0, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-message-backfill-retryable', + ownerA, + 'message graph timeout', + ) + + const afterFirst = await readParityRow(db, ownerA) + expect(afterFirst).toMatchObject({ + checkedAt: now.toISOString(), + messageCursorId: messageIds[mailboxParityMessagePageSize - 2], + messagesCompletedAt: null, + eventsCompletedAt: null, + contentWatermarkAt: now.toISOString(), + matchingSince: null, + mismatchCount: 2, + lastError: 'message backfill message graph timeout', + }) + + // Finish creation; count mismatch resets BOTH cursors + completion markers. + graphCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mocks.recordMailboxParityEvent.mockClear() + await parkUser(db, ownerB) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + const mismatch = await reconcileMailboxParity({ + env, + now: mismatchNow, + batchSize: 1, + }) + expect(mismatch).toEqual({ + scanned: 1, + backfilled: 4, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'evt-bound-1' }), + ) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, eventId: 'evt-orphan-1' }), + ) + + const afterMismatch = await readParityRow(db, ownerA) + expect(afterMismatch).toMatchObject({ + checkedAt: mismatchNow.toISOString(), + matchingSince: null, + mismatchCount: 3, + contentWatermarkAt: null, + contentReplayUpperAt: null, + contentReplayCursorId: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + lastError: null, + }) + + // Full historical rescan after cursor reset, including a live new row. + await insertMessage({ + db, + userId: ownerA, + id: 'msg-live-new', + createdAt: '2026-07-03T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mockDoCounts({ + threads: 1, + messages: mailboxParityMessagePageSize + 2, + attachments: 0, + deliveryEvents: 2, + }) + await parkUser(db, ownerB) + const rematchNow = new Date('2026-08-01T12:00:00.000Z') + const rematch = await reconcileMailboxParity({ + env, + now: rematchNow, + batchSize: 1, + }) + expect(rematch).toEqual({ + scanned: 1, + backfilled: mailboxParityMessagePageSize + 4, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith( + expect.objectContaining({ userId: ownerA, messageId: 'msg-live-new' }), + ) + const afterRematch = await readParityRow(db, ownerA) + expect(afterRematch).toMatchObject({ + matchingSince: rematchNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + messagesCompletedAt: expect.any(String), + eventsCompletedAt: expect.any(String), + messageCursorId: 'msg-live-new', + eventCursorId: 'evt-orphan-1', + }) + if (!afterRematch?.contentWatermarkAt) { + throw new Error('Expected content watermark after parity rematch.') + } + expect(afterRematch.contentWatermarkAt >= rematchNow.toISOString()).toBe(true) + + await parkUser(db, ownerB) + const soakNow = new Date('2026-08-02T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterSoak = await readParityRow(db, ownerA) + expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) + expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) + expect(afterSoak?.mismatchCount).toBe(0) + + consoleWarn.mockClear() + mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( + new Error('count boom\nwith\tcontrol'), + ) + await parkUser(db, ownerB) + const errorNow = new Date('2026-08-02T13:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: errorNow, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterError = await readParityRow(db, ownerA) + expect(afterError?.checkedAt).toBe(errorNow.toISOString()) + expect(afterError?.lastError).toBe('count boom with control') + expect(afterError?.matchingSince).toBeNull() +}) + +test('mailbox parity content watermark repairs equal-count updates and soak', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'd'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-a', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-b', + createdAt: '2026-07-01T00:00:01.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + now.toISOString(), + ) + + const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id IN ('msg-a', 'msg-b')`, + ) + .bind(sharedUpdatedAt) + .run() + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + const repairNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(repairNow) + await expect( + reconcileMailboxParity({ env, now: repairNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 2, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-a', 'msg-b']) + const afterRepair = await readParityRow(db, owner) + expect(afterRepair?.matchingSince).toBe(now.toISOString()) + expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) + expect(afterRepair?.contentReplayUpperAt).toBeNull() + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity replays message updated during initial backfill', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'e'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + const midUpdateAt = new Date('2026-08-01T10:00:05.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-early', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-late', + createdAt: '2026-07-01T00:00:01.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + if (input.messageId === 'msg-early') { + vi.setSystemTime(midUpdateAt) + await db + .prepare( + `UPDATE email_messages + SET classification = 'quarantined', updated_at = ? + WHERE id = 'msg-early'`, + ) + .bind(midUpdateAt.toISOString()) + .run() + } + return successGraph(input.messageId) + }, + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 3, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual(['msg-early', 'msg-late', 'msg-early']) + expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( + midUpdateAt.toISOString(), + ) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity content replay resumes durable upper and cursor across ticks', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'f'.repeat(64) + const baseline = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(baseline) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + const messageCount = mailboxParityContentPageSize + 2 + for (let index = 0; index < messageCount; index += 1) { + await insertMessage({ + db, + userId: owner, + id: `msg-${String(index).padStart(2, '0')}`, + createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: messageCount, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await reconcileMailboxParity({ env, now: baseline, batchSize: 1 }) + + const updateAt = '2026-08-01T10:30:00.000Z' + await db + .prepare(`UPDATE email_messages SET updated_at = ? WHERE user_id = ?`) + .bind(updateAt, owner) + .run() + + const replayNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(replayNow) + let contentCalls = 0 + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => { + contentCalls += 1 + if (contentCalls === mailboxParityContentPageSize) { + // Exhaust budget mid-window by freezing time past deadline. + vi.setSystemTime(new Date(replayNow.getTime() + 20_000)) + } + return successGraph(input.messageId) + }, + ) + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 0, + failed: 0, + backfilled: mailboxParityContentPageSize, + }) + const mid = await readParityRow(db, owner) + expect(mid).toMatchObject({ + contentWatermarkAt: baseline.toISOString(), + contentReplayUpperAt: replayNow.toISOString(), + contentReplayCursorId: `msg-${String(mailboxParityContentPageSize - 1).padStart(2, '0')}`, + matchingSince: null, + }) + + vi.setSystemTime(replayNow) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxMessageGraphFromD1.mockClear() + await expect( + reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), + ).resolves.toMatchObject({ + compared: 1, + matched: 1, + backfilled: 2, + }) + expect( + mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( + (call) => call[0].messageId as string, + ), + ).toEqual([ + `msg-${String(mailboxParityContentPageSize).padStart(2, '0')}`, + `msg-${String(mailboxParityContentPageSize + 1).padStart(2, '0')}`, + ]) + const done = await readParityRow(db, owner) + expect(done).toMatchObject({ + contentWatermarkAt: replayNow.toISOString(), + contentReplayUpperAt: null, + contentReplayCursorId: null, + // Soak was cleared on the incomplete tick; exact compare starts a new window. + matchingSince: replayNow.toISOString(), + }) + } finally { + vi.useRealTimers() + } +}) + +test('mailbox parity eventsTruncated still completes via full event phase', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'h'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + for (let index = 0; index < 3; index += 1) { + await insertEvent({ + db, + userId: owner, + id: `evt-${index}`, + messageId: 'msg-1', + createdAt: `2026-07-01T00:01:0${index}.000Z`, + }) + } + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue( + successGraph('msg-1', true), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 3, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 4, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledTimes(3) +}) diff --git a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts new file mode 100644 index 0000000000..6b8f9bd795 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts @@ -0,0 +1,423 @@ +import { + createParityDb, + getMailboxReconcileTestMocks, + insertEvent, + insertMessage, + insertUser, + listUsersForMailboxParity, + mockDoCounts, + parityEnv, + readParityRow, + reconcileMailboxParity, + successGraph, +} from './mailbox-reconcile-test-support.ts' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' + +const mocks = getMailboxReconcileTestMocks() + +test('mailbox parity missing and unconfigured do not advance creation cursors', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'g'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertMessage({ + db, + userId: owner, + id: 'msg-2', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ + messageId: 'msg-1', + message: { status: 'missing' as const }, + events: [], + eventsTruncated: false, + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 1, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ failed: 1, backfilled: 0, compared: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + messageCursorId: null, + messagesCompletedAt: null, + contentWatermarkAt: now.toISOString(), + lastError: 'message backfill message graph missing', + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValueOnce({ + status: 'skipped', + reason: 'mailbox-unconfigured', + }) + await expect( + reconcileMailboxParity({ + env, + now: new Date('2026-08-01T11:00:00.000Z'), + batchSize: 1, + }), + ).resolves.toMatchObject({ failed: 1, compared: 0 }) + const afterUnconfigured = await readParityRow(db, owner) + expect(afterUnconfigured).toMatchObject({ + messagesCompletedAt: expect.any(String), + eventCursorId: null, + eventsCompletedAt: null, + lastError: 'event backfill event skipped:mailbox-unconfigured', + }) +}) + +test('mailbox parity event missing advances cursor while unconfigured does not', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'j'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-gone', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-ok', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:02.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1 + .mockResolvedValueOnce({ status: 'missing' }) + .mockResolvedValueOnce({ status: 'mirrored' }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 2, + }) + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toMatchObject({ matched: 1, failed: 0 }) + expect(await readParityRow(db, owner)).toMatchObject({ + eventCursorId: 'evt-ok', + eventsCompletedAt: expect.any(String), + }) +}) + +test('mailbox parity deletion race purges Mailbox and skips parity writes', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'i'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + })), + purge, + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + try { + return { ok: true as const, value: await promise } + } catch { + return { ok: false as const, result: { status: 'timeout' as const } } + } + }, + ) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { + await db + .prepare(`UPDATE users SET deleting_at = ? WHERE stable_user_id = ?`) + .bind(now.toISOString(), owner) + .run() + return successGraph('msg-1') + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + + await expect( + reconcileMailboxParity({ + env: parityEnv(db), + now, + batchSize: 1, + }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const row = await readParityRow(db, owner) + // Baseline may have been written before deletion; no completion/compare state. + expect(row?.messagesCompletedAt).toBeNull() + expect(row?.matchingSince).toBeNull() + expect(row?.eventsCompletedAt).toBeNull() +}) + +test('tracked empty user discovers DO-only leftover, purges, then reaches zero parity', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'k'.repeat(64) + const neverTracked = 'l'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await insertUser({ db, userId: owner, checkedAt: null }) + await insertUser({ db, userId: neverTracked, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + const purge = vi.fn(async () => ({ ok: true as const })) + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + + // Simulate failed delete mirror: D1 cleared, DO still holds message/event. + await db + .prepare(`DELETE FROM email_delivery_events WHERE user_id = ?`) + .bind(owner) + .run() + await db + .prepare(`DELETE FROM email_messages WHERE user_id = ?`) + .bind(owner) + .run() + + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).toEqual([owner]) + expect( + (await listUsersForMailboxParity({ db, limit: 10 })).map( + (row) => row.userId, + ), + ).not.toContain(neverTracked) + + mockDoCounts( + { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + { purge }, + ) + const mismatchNow = new Date('2026-08-01T11:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect(purge).toHaveBeenCalledTimes(1) + const afterPurge = await readParityRow(db, owner) + expect(afterPurge).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + contentWatermarkAt: null, + messagesCompletedAt: null, + eventsCompletedAt: null, + messageCursorId: null, + eventCursorId: null, + lastError: null, + }) + + mockDoCounts({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + const zeroNow = new Date('2026-08-01T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const afterZero = await readParityRow(db, owner) + expect(afterZero).toMatchObject({ + matchingSince: zeroNow.toISOString(), + mismatchCount: 0, + contentWatermarkAt: expect.any(String), + }) + if (!afterZero?.contentWatermarkAt) { + throw new Error('Expected content watermark after zero-count parity.') + } + expect(afterZero.contentWatermarkAt >= zeroNow.toISOString()).toBe(true) +}) + +test('mismatch purge failure clears soak and does not start rebuild reset', async () => { + consoleWarn.mockImplementation(() => {}) + vi.useFakeTimers() + const { db } = await createParityDb() + const owner = 'm'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + vi.setSystemTime(now) + + try { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored', + }) + mockDoCounts({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1 }) + const afterMatch = await readParityRow(db, owner) + expect(afterMatch?.messagesCompletedAt).toEqual(expect.any(String)) + expect(afterMatch?.contentWatermarkAt).toBe(now.toISOString()) + + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => ({ + threads: 0, + messages: 2, + attachments: 0, + deliveryEvents: 0, + })), + purge: vi.fn(async () => ({ ok: true as const })), + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => { + const settled = await promise + if (settled && typeof settled === 'object' && 'messages' in settled) { + return { ok: true as const, value: settled } + } + return { ok: false as const, timedOut: true as const } + }, + ) + consoleWarn.mockClear() + const failNow = new Date('2026-08-01T11:00:00.000Z') + vi.setSystemTime(failNow) + await expect( + reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 0, + mismatched: 1, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-mismatch-purge-failed', + owner, + 'timeout', + 'mailbox metadata purge timeout', + ) + const afterFail = await readParityRow(db, owner) + // Empty content window may advance the watermark before compare; purge + // failure must not clear creation completion/cursors (rebuild not started). + expect(afterFail).toMatchObject({ + matchingSince: null, + mismatchCount: 1, + messagesCompletedAt: afterMatch?.messagesCompletedAt, + eventsCompletedAt: afterMatch?.eventsCompletedAt, + messageCursorId: afterMatch?.messageCursorId, + lastError: 'mailbox metadata purge timeout', + checkedAt: failNow.toISOString(), + }) + expect(afterFail?.contentWatermarkAt).not.toBeNull() + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-reconcile-test-support.ts b/packages/worker/src/email/mailbox-reconcile-test-support.ts new file mode 100644 index 0000000000..2baf8e1cff --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile-test-support.ts @@ -0,0 +1,219 @@ +import { DatabaseSync } from 'node:sqlite' +import { vi } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import type * as MailboxMirrorModule from './mailbox-mirror.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const mocks = vi.hoisted(() => ({ + mirrorMailboxMessageGraphFromD1: vi.fn(), + mirrorMailboxDeliveryEventFromD1: vi.fn(), + awaitMailboxMirrorRpc: vi.fn(), + recordMailboxParityEvent: vi.fn(), + mailboxRpc: vi.fn(), +})) + +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, + mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, +})) + +vi.mock('./mailbox-mirror.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + awaitMailboxMirrorRpc: mocks.awaitMailboxMirrorRpc, + } +}) + +vi.mock('./mailbox-parity-events.ts', () => ({ + recordMailboxParityEvent: mocks.recordMailboxParityEvent, +})) + +vi.mock('./mailbox-client.ts', () => ({ + mailboxRpc: mocks.mailboxRpc, +})) + +export type MailboxReconcileTestMocks = typeof mocks + +export function getMailboxReconcileTestMocks(): MailboxReconcileTestMocks { + return mocks +} + +export const { + countD1MailboxParity, + listEventBackfillPage, + listUsersForMailboxParity, + reconcileMailboxParity, +} = await import('./mailbox-reconcile.ts') + +export async function createParityDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureUsersTestSchema({ db }) + await ensureEmailTestSchema(db) + return { sqlite, db } +} + +export async function insertUser(input: { + db: D1Database + userId: string + email?: string + deletingAt?: string | null + checkedAt?: string | null +}) { + await input.db + .prepare( + `INSERT INTO users ( + username, email, password_hash, stable_user_id, deleting_at, + mailbox_parity_checked_at + ) VALUES (?, ?, 'hash', ?, ?, ?)`, + ) + .bind( + `u-${crypto.randomUUID().slice(0, 8)}`, + input.email ?? `${crypto.randomUUID()}@example.test`, + input.userId, + input.deletingAt ?? null, + input.checkedAt ?? null, + ) + .run() +} + +export async function insertMessage(input: { + db: D1Database + userId: string + id: string + createdAt: string + updatedAt?: string +}) { + await input.db + .prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, text_body, raw_size, + processing_status, created_at, updated_at + ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, + 'stored', ?, ?)`, + ) + .bind( + input.id, + input.userId, + input.createdAt, + input.updatedAt ?? input.createdAt, + ) + .run() +} + +export async function insertEvent(input: { + db: D1Database + userId: string + id: string + createdAt: string + messageId?: string | null +}) { + await input.db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, ?, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind( + input.id, + input.messageId === undefined ? null : input.messageId, + input.userId, + input.createdAt, + ) + .run() +} + +export async function parkUser(db: D1Database, userId: string) { + await db + .prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-01-01T00:00:00.000Z' + WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() +} + +export async function readParityRow(db: D1Database, userId: string) { + return db + .prepare( + `SELECT + mailbox_parity_checked_at AS checkedAt, + mailbox_parity_matching_since AS matchingSince, + mailbox_parity_mismatch_count AS mismatchCount, + mailbox_parity_last_error AS lastError, + mailbox_parity_content_watermark_at AS contentWatermarkAt, + mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, + mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, + mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, + mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, + mailbox_parity_message_backfill_cursor_id AS messageCursorId, + mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, + mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, + mailbox_parity_event_backfill_cursor_id AS eventCursorId, + mailbox_parity_event_backfill_completed_at AS eventsCompletedAt + FROM users + WHERE stable_user_id = ?`, + ) + .bind(userId) + .first<{ + checkedAt: string | null + matchingSince: string | null + mismatchCount: number + lastError: string | null + contentWatermarkAt: string | null + contentReplayUpperAt: string | null + contentReplayCursorUpdatedAt: string | null + contentReplayCursorId: string | null + messageCursorCreatedAt: string | null + messageCursorId: string | null + messagesCompletedAt: string | null + eventCursorCreatedAt: string | null + eventCursorId: string | null + eventsCompletedAt: string | null + }>() +} + +export function successGraph(messageId: string, eventsTruncated = false) { + return { + messageId, + message: { status: 'mirrored' as const }, + events: [], + eventsTruncated, + } +} + +export function mockDoCounts( + counts: { + threads: number + messages: number + attachments: number + deliveryEvents: number + }, + options?: { purge?: () => Promise<{ ok: true }> }, +) { + const purge = options?.purge ?? vi.fn(async () => ({ ok: true as const })) + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => counts), + purge, + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => ({ + ok: true as const, + value: await promise, + }), + ) + return { purge } +} + +export function parityEnv(db: D1Database) { + return { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } +} diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index c1cea9693b..b2b1dec2dd 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -17,6 +17,8 @@ import { mailboxParityMessagePageSize, mirrorMailboxParityDeliveryEventFromD1, replayContentUpdatesForUser, + type MailboxParityContentPhaseResult, + type MailboxParityCreationPhaseResult, } from './mailbox-parity-phases.ts' import { countD1MailboxParity, @@ -71,6 +73,32 @@ export { listUsersForMailboxParity, } +type MailboxParityCompareResult = + | { status: 'matched'; state: MailboxParityUserState } + | { status: 'mismatched_reset'; state: MailboxParityUserState } + | { + status: 'mismatched_purge_failed' + state: MailboxParityUserState + lastError: string + } + +type MailboxParityUserTickResult = + | { status: 'skipped' } + | { status: 'deletion_stopped'; backfilled: number } + | { + status: 'phase_checkpoint' + backfilled: number + failed: boolean + budgetExhausted: boolean + } + | { + status: 'compared' + backfilled: number + matched: boolean + mismatched: boolean + failed: boolean + } + function safeParityErrorText(error: unknown): string { const raw = error instanceof Error ? error.message : String(error) let cleaned = '' @@ -214,12 +242,7 @@ async function compareParityForUser(input: { env: MailboxParityReconcileEnv state: MailboxParityUserState nowIso: string -}): Promise<{ - state: MailboxParityUserState - matched: boolean - purgeFailed: boolean - lastError: string | null -}> { +}): Promise { const d1 = await countD1MailboxParity({ db: input.env.APP_DB, userId: input.state.userId, @@ -241,14 +264,12 @@ async function compareParityForUser(input: { }) if (matched) { return { + status: 'matched', state: { ...input.state, matchingSince: input.state.matchingSince ?? input.nowIso, mismatchCount: 0, }, - matched: true, - purgeFailed: false, - lastError: null, } } @@ -266,18 +287,18 @@ async function compareParityForUser(input: { purged.errorText, ) return { + status: 'mismatched_purge_failed', state: { ...input.state, matchingSince: null, mismatchCount: input.state.mismatchCount + 1, }, - matched: false, - purgeFailed: true, lastError: purged.errorText, } } return { + status: 'mismatched_reset', state: { ...input.state, matchingSince: null, @@ -290,9 +311,85 @@ async function compareParityForUser(input: { contentReplayUpperAt: null, contentReplayCursor: null, }, - matched: false, - purgeFailed: false, - lastError: null, + } +} + +function creationPhaseCheckpoint(input: { + phase: 'message' | 'event' + result: MailboxParityCreationPhaseResult +}): { + kind: 'continue' | 'deletion' | 'failed' | 'budget' + lastError: string | null + warnDetail: string | null +} { + switch (input.result.status) { + case 'complete': + return { kind: 'continue', lastError: null, warnDetail: null } + case 'deletion_started': + return { kind: 'deletion', lastError: null, warnDetail: null } + case 'budget_exhausted': + return { kind: 'budget', lastError: null, warnDetail: null } + case 'retryable_failure': { + const prefix = + input.phase === 'message' ? 'message backfill' : 'event backfill' + return { + kind: 'failed', + lastError: `${prefix} ${input.result.blockedReason}`, + warnDetail: input.result.blockedReason, + } + } + default: { + const exhaustive: never = input.result + throw new Error( + `Unhandled creation phase status: ${JSON.stringify(exhaustive)}`, + ) + } + } +} + +function contentPhaseCheckpoint(input: { + result: MailboxParityContentPhaseResult +}): { + kind: 'continue' | 'deletion' | 'failed' | 'budget' + lastError: string | null + warnDetail: string | null + freezeOpenedWindow: boolean +} { + switch (input.result.status) { + case 'complete': + return { + kind: 'continue', + lastError: null, + warnDetail: null, + freezeOpenedWindow: false, + } + case 'deletion_started': + return { + kind: 'deletion', + lastError: null, + warnDetail: null, + freezeOpenedWindow: false, + } + case 'budget_exhausted': + return { + kind: 'budget', + lastError: null, + warnDetail: null, + freezeOpenedWindow: input.result.openedWindow, + } + case 'retryable_failure': + return { + kind: 'failed', + lastError: `content replay ${input.result.blockedReason}`, + warnDetail: input.result.blockedReason, + freezeOpenedWindow: input.result.openedWindow, + } + default: { + const exhaustive: never = input.result + throw new Error( + `Unhandled content phase status: ${JSON.stringify(exhaustive)}`, + ) + } } } @@ -301,27 +398,13 @@ async function reconcileOneUser(input: { userId: string nowIso: string deadlineMs: number -}): Promise<{ - backfilled: number - compared: boolean - matched: boolean - mismatched: boolean - failed: boolean - budgetExhausted: boolean -}> { +}): Promise { const loaded = await loadUserParityState({ db: input.env.APP_DB, userId: input.userId, }) if (!loaded) { - return { - backfilled: 0, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: false, - } + return { status: 'skipped' } } let state = loaded @@ -331,30 +414,23 @@ async function reconcileOneUser(input: { const isDeleting = () => isUserDeleting({ db: input.env.APP_DB, userId: state.userId }) - const stopForDeletion = async () => { + const stopForDeletion = async (): Promise => { await purgeMailboxBestEffort({ env: input.env, userId: state.userId, }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: false, - } + return { status: 'deletion_stopped', backfilled } } - const persistProgress = async (inputPersist: { + const persistCheckpoint = async (checkpoint: { matchingSince: string | null lastError: string | null }) => { - const { wrote } = await persistUserParityProgress({ + await persistUserParityProgress({ db: input.env.APP_DB, userId: state.userId, nowIso: input.nowIso, - matchingSince: inputPersist.matchingSince, + matchingSince: checkpoint.matchingSince, mismatchCount: state.mismatchCount, contentWatermarkAt: state.contentWatermarkAt, contentReplayUpperAt: state.contentReplayUpperAt, @@ -363,9 +439,8 @@ async function reconcileOneUser(input: { messagesCompletedAt: state.messagesCompletedAt, eventCursor: state.eventCursor, eventsCompletedAt: state.eventsCompletedAt, - lastError: inputPersist.lastError, + lastError: checkpoint.lastError, }) - return wrote } if (await isDeleting()) { @@ -391,39 +466,42 @@ async function reconcileOneUser(input: { state = messagePass.state creationBackfilled += messagePass.backfilled backfilled += messagePass.backfilled - if (messagePass.deletionStarted) { - return stopForDeletion() - } - if (messagePass.retryableFailure) { - await persistProgress({ - matchingSince: null, - lastError: messagePass.blockedReason - ? `message backfill ${messagePass.blockedReason}` - : 'message backfill mirror error or timeout', - }) - console.warn( - 'mailbox-parity-message-backfill-retryable', - state.userId, - messagePass.blockedReason ?? 'error-or-timeout', - ) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: true, - budgetExhausted: false, - } - } - if (messagePass.budgetExhausted) { - await persistProgress({ matchingSince: null, lastError: null }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: true, + const messageCheckpoint = creationPhaseCheckpoint({ + phase: 'message', + result: messagePass, + }) + switch (messageCheckpoint.kind) { + case 'continue': + break + case 'deletion': + return stopForDeletion() + case 'failed': + await persistCheckpoint({ + matchingSince: null, + lastError: messageCheckpoint.lastError, + }) + console.warn( + 'mailbox-parity-message-backfill-retryable', + state.userId, + messageCheckpoint.warnDetail ?? 'error-or-timeout', + ) + return { + status: 'phase_checkpoint', + backfilled, + failed: true, + budgetExhausted: false, + } + case 'budget': + await persistCheckpoint({ matchingSince: null, lastError: null }) + return { + status: 'phase_checkpoint', + backfilled, + failed: false, + budgetExhausted: true, + } + default: { + const exhaustive: never = messageCheckpoint.kind + throw new Error(`Unhandled message checkpoint: ${exhaustive}`) } } @@ -436,49 +514,50 @@ async function reconcileOneUser(input: { state = eventPass.state creationBackfilled += eventPass.backfilled backfilled += eventPass.backfilled - if (eventPass.deletionStarted) { - return stopForDeletion() - } - if (eventPass.retryableFailure) { - await persistProgress({ - matchingSince: null, - lastError: eventPass.blockedReason - ? `event backfill ${eventPass.blockedReason}` - : 'event backfill mirror error or timeout', - }) - console.warn( - 'mailbox-parity-event-backfill-retryable', - state.userId, - eventPass.blockedReason ?? 'error-or-timeout', - ) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: true, - budgetExhausted: false, - } - } - if (eventPass.budgetExhausted) { - await persistProgress({ matchingSince: null, lastError: null }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: true, + const eventCheckpoint = creationPhaseCheckpoint({ + phase: 'event', + result: eventPass, + }) + switch (eventCheckpoint.kind) { + case 'continue': + break + case 'deletion': + return stopForDeletion() + case 'failed': + await persistCheckpoint({ + matchingSince: null, + lastError: eventCheckpoint.lastError, + }) + console.warn( + 'mailbox-parity-event-backfill-retryable', + state.userId, + eventCheckpoint.warnDetail ?? 'error-or-timeout', + ) + return { + status: 'phase_checkpoint', + backfilled, + failed: true, + budgetExhausted: false, + } + case 'budget': + await persistCheckpoint({ matchingSince: null, lastError: null }) + return { + status: 'phase_checkpoint', + backfilled, + failed: false, + budgetExhausted: true, + } + default: { + const exhaustive: never = eventCheckpoint.kind + throw new Error(`Unhandled event checkpoint: ${exhaustive}`) } } if (state.messagesCompletedAt == null || state.eventsCompletedAt == null) { - await persistProgress({ matchingSince: null, lastError: null }) + await persistCheckpoint({ matchingSince: null, lastError: null }) return { + status: 'phase_checkpoint', backfilled, - compared: false, - matched: false, - mismatched: false, failed: false, budgetExhausted: false, } @@ -499,47 +578,47 @@ async function reconcileOneUser(input: { }) state = contentPass.state backfilled += contentPass.backfilled - if (contentPass.deletionStarted) { - return stopForDeletion() - } - // Durably freeze a newly opened window even when this tick cannot finish it. - if (contentPass.openedWindow && !contentPass.windowComplete) { - await persistProgress({ - matchingSince: null, - lastError: null, - }) - } - if (contentPass.retryableFailure) { - await persistProgress({ - matchingSince: null, - lastError: contentPass.blockedReason - ? `content replay ${contentPass.blockedReason}` - : 'content replay mirror error or timeout', - }) - console.warn( - 'mailbox-parity-content-replay-retryable', - state.userId, - contentPass.blockedReason ?? 'error-or-timeout', - ) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: true, - budgetExhausted: false, - } - } - if (!contentPass.windowComplete) { - // Freeze/retain upper + cursor; clear soak on incomplete budget ticks. - await persistProgress({ matchingSince: null, lastError: null }) - return { - backfilled, - compared: false, - matched: false, - mismatched: false, - failed: false, - budgetExhausted: contentPass.budgetExhausted, + const contentCheckpoint = contentPhaseCheckpoint({ result: contentPass }) + switch (contentCheckpoint.kind) { + case 'continue': + break + case 'deletion': + return stopForDeletion() + case 'failed': + // Durably freeze a newly opened window even when this tick cannot finish it. + if (contentCheckpoint.freezeOpenedWindow) { + await persistCheckpoint({ matchingSince: null, lastError: null }) + } + await persistCheckpoint({ + matchingSince: null, + lastError: contentCheckpoint.lastError, + }) + console.warn( + 'mailbox-parity-content-replay-retryable', + state.userId, + contentCheckpoint.warnDetail ?? 'error-or-timeout', + ) + return { + status: 'phase_checkpoint', + backfilled, + failed: true, + budgetExhausted: false, + } + case 'budget': + if (contentCheckpoint.freezeOpenedWindow) { + await persistCheckpoint({ matchingSince: null, lastError: null }) + } + // Freeze/retain upper + cursor; clear soak on incomplete budget ticks. + await persistCheckpoint({ matchingSince: null, lastError: null }) + return { + status: 'phase_checkpoint', + backfilled, + failed: false, + budgetExhausted: true, + } + default: { + const exhaustive: never = contentCheckpoint.kind + throw new Error(`Unhandled content checkpoint: ${exhaustive}`) } } @@ -556,17 +635,48 @@ async function reconcileOneUser(input: { if (await isDeleting()) { return stopForDeletion() } - await persistProgress({ - matchingSince: state.matchingSince, - lastError: compared.lastError, - }) - return { - backfilled, - compared: true, - matched: compared.matched, - mismatched: !compared.matched, - failed: compared.purgeFailed, - budgetExhausted: false, + + switch (compared.status) { + case 'matched': + await persistCheckpoint({ + matchingSince: state.matchingSince, + lastError: null, + }) + return { + status: 'compared', + backfilled, + matched: true, + mismatched: false, + failed: false, + } + case 'mismatched_reset': + await persistCheckpoint({ + matchingSince: state.matchingSince, + lastError: null, + }) + return { + status: 'compared', + backfilled, + matched: false, + mismatched: true, + failed: false, + } + case 'mismatched_purge_failed': + await persistCheckpoint({ + matchingSince: state.matchingSince, + lastError: compared.lastError, + }) + return { + status: 'compared', + backfilled, + matched: false, + mismatched: true, + failed: true, + } + default: { + const exhaustive: never = compared + throw new Error(`Unhandled compare status: ${JSON.stringify(exhaustive)}`) + } } } @@ -603,12 +713,34 @@ export async function reconcileMailboxParity(input: { nowIso, deadlineMs, }) - metrics.backfilled += result.backfilled - if (result.compared) metrics.compared += 1 - if (result.matched) metrics.matched += 1 - if (result.mismatched) metrics.mismatched += 1 - if (result.failed) metrics.failed += 1 - if (result.budgetExhausted) break + switch (result.status) { + case 'skipped': + break + case 'deletion_stopped': + metrics.backfilled += result.backfilled + break + case 'phase_checkpoint': + metrics.backfilled += result.backfilled + if (result.failed) metrics.failed += 1 + if (result.budgetExhausted) { + // Stop scanning further users this tick. + return metrics + } + break + case 'compared': + metrics.backfilled += result.backfilled + metrics.compared += 1 + if (result.matched) metrics.matched += 1 + if (result.mismatched) metrics.mismatched += 1 + if (result.failed) metrics.failed += 1 + break + default: { + const exhaustive: never = result + throw new Error( + `Unhandled user tick status: ${JSON.stringify(exhaustive)}`, + ) + } + } } catch (error) { metrics.failed += 1 const lastError = safeParityErrorText(error) From a09e7c6e73a6017d47cb087e447a0e3b48bfe89e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:35:38 +0000 Subject: [PATCH 17/62] test(email): remove superseded parity suite Co-authored-by: Kent C. Dodds --- .../src/email/mailbox-reconcile.node.test.ts | 1226 ----------------- 1 file changed, 1226 deletions(-) delete mode 100644 packages/worker/src/email/mailbox-reconcile.node.test.ts diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts deleted file mode 100644 index f082b68e95..0000000000 --- a/packages/worker/src/email/mailbox-reconcile.node.test.ts +++ /dev/null @@ -1,1226 +0,0 @@ -import { DatabaseSync } from 'node:sqlite' -import { expect, test, vi } from 'vitest' -import { consoleWarn } from '#worker/test-support/console-spies.ts' -import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' -import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' -import { systemEmailOwnerId } from './email-owner.ts' -import type * as MailboxMirrorModule from './mailbox-mirror.ts' -import { - mailboxParityContentPageSize, - mailboxParityMessagePageSize, -} from './mailbox-parity-phases.ts' -import { ensureEmailTestSchema } from './test-schema.ts' - -const mocks = vi.hoisted(() => ({ - mirrorMailboxMessageGraphFromD1: vi.fn(), - mirrorMailboxDeliveryEventFromD1: vi.fn(), - awaitMailboxMirrorRpc: vi.fn(), - recordMailboxParityEvent: vi.fn(), - mailboxRpc: vi.fn(), -})) - -vi.mock('./mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, - mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, -})) - -vi.mock('./mailbox-mirror.ts', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - awaitMailboxMirrorRpc: mocks.awaitMailboxMirrorRpc, - } -}) - -vi.mock('./mailbox-parity-events.ts', () => ({ - recordMailboxParityEvent: mocks.recordMailboxParityEvent, -})) - -vi.mock('./mailbox-client.ts', () => ({ - mailboxRpc: mocks.mailboxRpc, -})) - -const { - countD1MailboxParity, - listEventBackfillPage, - listUsersForMailboxParity, - reconcileMailboxParity, -} = await import('./mailbox-reconcile.ts') - -async function createParityDb() { - const sqlite = new DatabaseSync(':memory:') - const db = createD1FromSqlite(sqlite) - await ensureUsersTestSchema({ db }) - await ensureEmailTestSchema(db) - return { sqlite, db } -} - -async function insertUser(input: { - db: D1Database - userId: string - email?: string - deletingAt?: string | null - checkedAt?: string | null -}) { - await input.db - .prepare( - `INSERT INTO users ( - username, email, password_hash, stable_user_id, deleting_at, - mailbox_parity_checked_at - ) VALUES (?, ?, 'hash', ?, ?, ?)`, - ) - .bind( - `u-${crypto.randomUUID().slice(0, 8)}`, - input.email ?? `${crypto.randomUUID()}@example.test`, - input.userId, - input.deletingAt ?? null, - input.checkedAt ?? null, - ) - .run() -} - -async function insertMessage(input: { - db: D1Database - userId: string - id: string - createdAt: string - updatedAt?: string -}) { - await input.db - .prepare( - `INSERT INTO email_messages ( - id, direction, user_id, from_address, subject, text_body, raw_size, - processing_status, created_at, updated_at - ) VALUES (?, 'inbound', ?, 'from@example.test', 'hi', 'body', 4, - 'stored', ?, ?)`, - ) - .bind( - input.id, - input.userId, - input.createdAt, - input.updatedAt ?? input.createdAt, - ) - .run() -} - -async function insertEvent(input: { - db: D1Database - userId: string - id: string - createdAt: string - messageId?: string | null -}) { - await input.db - .prepare( - `INSERT INTO email_delivery_events ( - id, message_id, user_id, event_type, provider, detail_json, created_at - ) VALUES (?, ?, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, - ) - .bind( - input.id, - input.messageId === undefined ? null : input.messageId, - input.userId, - input.createdAt, - ) - .run() -} - -async function parkUser(db: D1Database, userId: string) { - await db - .prepare( - `UPDATE users - SET mailbox_parity_checked_at = '9999-01-01T00:00:00.000Z' - WHERE stable_user_id = ?`, - ) - .bind(userId) - .run() -} - -async function readParityRow(db: D1Database, userId: string) { - return db - .prepare( - `SELECT - mailbox_parity_checked_at AS checkedAt, - mailbox_parity_matching_since AS matchingSince, - mailbox_parity_mismatch_count AS mismatchCount, - mailbox_parity_last_error AS lastError, - mailbox_parity_content_watermark_at AS contentWatermarkAt, - mailbox_parity_content_replay_upper_at AS contentReplayUpperAt, - mailbox_parity_content_replay_cursor_updated_at AS contentReplayCursorUpdatedAt, - mailbox_parity_content_replay_cursor_id AS contentReplayCursorId, - mailbox_parity_message_backfill_cursor_created_at AS messageCursorCreatedAt, - mailbox_parity_message_backfill_cursor_id AS messageCursorId, - mailbox_parity_message_backfill_completed_at AS messagesCompletedAt, - mailbox_parity_event_backfill_cursor_created_at AS eventCursorCreatedAt, - mailbox_parity_event_backfill_cursor_id AS eventCursorId, - mailbox_parity_event_backfill_completed_at AS eventsCompletedAt - FROM users - WHERE stable_user_id = ?`, - ) - .bind(userId) - .first<{ - checkedAt: string | null - matchingSince: string | null - mismatchCount: number - lastError: string | null - contentWatermarkAt: string | null - contentReplayUpperAt: string | null - contentReplayCursorUpdatedAt: string | null - contentReplayCursorId: string | null - messageCursorCreatedAt: string | null - messageCursorId: string | null - messagesCompletedAt: string | null - eventCursorCreatedAt: string | null - eventCursorId: string | null - eventsCompletedAt: string | null - }>() -} - -function successGraph(messageId: string, eventsTruncated = false) { - return { - messageId, - message: { status: 'mirrored' as const }, - events: [], - eventsTruncated, - } -} - -function mockDoCounts( - counts: { - threads: number - messages: number - attachments: number - deliveryEvents: number - }, - options?: { purge?: () => Promise<{ ok: true }> }, -) { - const purge = options?.purge ?? vi.fn(async () => ({ ok: true as const })) - mocks.mailboxRpc.mockImplementation(() => ({ - countMailbox: vi.fn(async () => counts), - purge, - })) - mocks.awaitMailboxMirrorRpc.mockImplementation( - async (promise: Promise) => ({ - ok: true as const, - value: await promise, - }), - ) - return { purge } -} - -function parityEnv(db: D1Database) { - return { - APP_DB: db, - MAILBOX: {} as DurableObjectNamespace, - EMAIL_EVENTS: { - writeDataPoint: vi.fn(), - } as unknown as AnalyticsEngineDataset, - } -} - -test('mailbox parity multi-run cursor progress, isolation, soak, errors, and counts', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const ownerA = 'a'.repeat(64) - const ownerB = 'b'.repeat(64) - const deleting = 'c'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - - await insertUser({ db, userId: ownerA, checkedAt: null }) - await insertUser({ - db, - userId: ownerB, - checkedAt: '2026-07-01T00:00:00.000Z', - }) - await insertUser({ - db, - userId: deleting, - deletingAt: '2026-07-31T00:00:00.000Z', - checkedAt: null, - }) - - const messageIds = Array.from( - { length: mailboxParityMessagePageSize + 1 }, - (_, index) => `msg-${String(index + 1).padStart(2, '0')}`, - ) - for (const [index, id] of messageIds.entries()) { - await insertMessage({ - db, - userId: ownerA, - id, - createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, - }) - } - await insertEvent({ - db, - userId: ownerA, - id: 'evt-bound-1', - messageId: messageIds[0]!, - createdAt: '2026-07-02T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: ownerA, - id: 'evt-orphan-1', - createdAt: '2026-07-02T00:00:01.000Z', - }) - await insertMessage({ - db, - userId: ownerB, - id: 'b-msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: systemEmailOwnerId, - id: 'sys-msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: deleting, - id: 'del-evt', - createdAt: '2026-07-01T00:00:00.000Z', - }) - - const discovered = await listUsersForMailboxParity({ db, limit: 10 }) - expect(discovered.map((row) => row.userId)).toEqual([ownerA, ownerB]) - expect(discovered.map((row) => row.userId)).not.toContain(systemEmailOwnerId) - expect(discovered.map((row) => row.userId)).not.toContain(deleting) - - const eventPage = await listEventBackfillPage({ - db, - userId: ownerA, - cursor: null, - limit: 10, - }) - expect(eventPage.map((row) => row.id)).toEqual([ - 'evt-bound-1', - 'evt-orphan-1', - ]) - - await db - .prepare( - `INSERT INTO email_threads ( - id, user_id, subject_normalized, last_message_at, created_at, updated_at - ) VALUES ('thread-a', ?, 'hi', ?, ?, ?)`, - ) - .bind( - ownerA, - '2026-07-01T00:00:00.000Z', - '2026-07-01T00:00:00.000Z', - '2026-07-01T00:00:00.000Z', - ) - .run() - - await expect(countD1MailboxParity({ db, userId: ownerA })).resolves.toEqual({ - threads: 1, - messages: mailboxParityMessagePageSize + 1, - attachments: 0, - deliveryEvents: 2, - }) - - let graphCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - graphCalls += 1 - if (graphCalls === mailboxParityMessagePageSize) { - return { - messageId: input.messageId, - message: { status: 'timeout' as const }, - events: [], - eventsTruncated: false, - } - } - return successGraph(input.messageId) - }, - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 0, - attachments: 0, - deliveryEvents: 0, - }) - - const env = parityEnv(db) - await db - .prepare( - `UPDATE users - SET mailbox_parity_matching_since = '2026-07-01T00:00:00.000Z', - mailbox_parity_mismatch_count = 2 - WHERE stable_user_id = ?`, - ) - .bind(ownerA) - .run() - - await parkUser(db, ownerB) - const first = await reconcileMailboxParity({ - env, - now, - batchSize: 1, - }) - expect(first).toEqual({ - scanned: 1, - backfilled: mailboxParityMessagePageSize - 1, - compared: 0, - matched: 0, - mismatched: 0, - failed: 1, - }) - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-message-backfill-retryable', - ownerA, - 'message graph timeout', - ) - - const afterFirst = await readParityRow(db, ownerA) - expect(afterFirst).toMatchObject({ - checkedAt: now.toISOString(), - messageCursorId: messageIds[mailboxParityMessagePageSize - 2], - messagesCompletedAt: null, - eventsCompletedAt: null, - contentWatermarkAt: now.toISOString(), - matchingSince: null, - mismatchCount: 2, - lastError: 'message backfill message graph timeout', - }) - - // Finish creation; count mismatch resets BOTH cursors + completion markers. - graphCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - mocks.mirrorMailboxDeliveryEventFromD1.mockClear() - mocks.recordMailboxParityEvent.mockClear() - await parkUser(db, ownerB) - const mismatchNow = new Date('2026-08-01T11:00:00.000Z') - const mismatch = await reconcileMailboxParity({ - env, - now: mismatchNow, - batchSize: 1, - }) - expect(mismatch).toEqual({ - scanned: 1, - backfilled: 4, - compared: 1, - matched: 0, - mismatched: 1, - failed: 0, - }) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'evt-bound-1' }), - ) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'evt-orphan-1' }), - ) - - const afterMismatch = await readParityRow(db, ownerA) - expect(afterMismatch).toMatchObject({ - checkedAt: mismatchNow.toISOString(), - matchingSince: null, - mismatchCount: 3, - contentWatermarkAt: null, - contentReplayUpperAt: null, - contentReplayCursorId: null, - messagesCompletedAt: null, - eventsCompletedAt: null, - messageCursorId: null, - eventCursorId: null, - lastError: null, - }) - - // Full historical rescan after cursor reset, including a live new row. - await insertMessage({ - db, - userId: ownerA, - id: 'msg-live-new', - createdAt: '2026-07-03T00:00:00.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - mocks.mirrorMailboxDeliveryEventFromD1.mockClear() - mockDoCounts({ - threads: 1, - messages: mailboxParityMessagePageSize + 2, - attachments: 0, - deliveryEvents: 2, - }) - await parkUser(db, ownerB) - const rematchNow = new Date('2026-08-01T12:00:00.000Z') - const rematch = await reconcileMailboxParity({ - env, - now: rematchNow, - batchSize: 1, - }) - expect(rematch).toEqual({ - scanned: 1, - backfilled: mailboxParityMessagePageSize + 4, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect(mocks.mirrorMailboxMessageGraphFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, messageId: 'msg-live-new' }), - ) - const afterRematch = await readParityRow(db, ownerA) - expect(afterRematch).toMatchObject({ - matchingSince: rematchNow.toISOString(), - mismatchCount: 0, - contentWatermarkAt: expect.any(String), - messagesCompletedAt: expect.any(String), - eventsCompletedAt: expect.any(String), - messageCursorId: 'msg-live-new', - eventCursorId: 'evt-orphan-1', - }) - if (!afterRematch?.contentWatermarkAt) { - throw new Error('Expected content watermark after parity rematch.') - } - expect(afterRematch.contentWatermarkAt >= rematchNow.toISOString()).toBe(true) - - await parkUser(db, ownerB) - const soakNow = new Date('2026-08-02T12:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: soakNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - const afterSoak = await readParityRow(db, ownerA) - expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) - expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) - expect(afterSoak?.mismatchCount).toBe(0) - - consoleWarn.mockClear() - mocks.awaitMailboxMirrorRpc.mockRejectedValueOnce( - new Error('count boom\nwith\tcontrol'), - ) - await parkUser(db, ownerB) - const errorNow = new Date('2026-08-02T13:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: errorNow, batchSize: 1 }), - ).resolves.toMatchObject({ failed: 1, compared: 0 }) - const afterError = await readParityRow(db, ownerA) - expect(afterError?.checkedAt).toBe(errorNow.toISOString()) - expect(afterError?.lastError).toBe('count boom with control') - expect(afterError?.matchingSince).toBeNull() -}) - -test('mailbox parity content watermark repairs equal-count updates and soak', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'd'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - vi.setSystemTime(now) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-a', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: owner, - id: 'msg-b', - createdAt: '2026-07-01T00:00:01.000Z', - }) - - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 0, - }) - const env = parityEnv(db) - - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ matched: 1, compared: 1 }) - expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( - now.toISOString(), - ) - - const sharedUpdatedAt = '2026-08-01T10:30:00.000Z' - await db - .prepare( - `UPDATE email_messages - SET classification = 'quarantined', updated_at = ? - WHERE id IN ('msg-a', 'msg-b')`, - ) - .bind(sharedUpdatedAt) - .run() - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - const repairNow = new Date('2026-08-01T11:00:00.000Z') - vi.setSystemTime(repairNow) - await expect( - reconcileMailboxParity({ env, now: repairNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 2, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ), - ).toEqual(['msg-a', 'msg-b']) - const afterRepair = await readParityRow(db, owner) - expect(afterRepair?.matchingSince).toBe(now.toISOString()) - expect(afterRepair?.contentWatermarkAt).toBe(repairNow.toISOString()) - expect(afterRepair?.contentReplayUpperAt).toBeNull() - } finally { - vi.useRealTimers() - } -}) - -test('mailbox parity replays message updated during initial backfill', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'e'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - const midUpdateAt = new Date('2026-08-01T10:00:05.000Z') - vi.setSystemTime(now) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-early', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: owner, - id: 'msg-late', - createdAt: '2026-07-01T00:00:01.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - if (input.messageId === 'msg-early') { - vi.setSystemTime(midUpdateAt) - await db - .prepare( - `UPDATE email_messages - SET classification = 'quarantined', updated_at = ? - WHERE id = 'msg-early'`, - ) - .bind(midUpdateAt.toISOString()) - .run() - } - return successGraph(input.messageId) - }, - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 0, - }) - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 3, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ), - ).toEqual(['msg-early', 'msg-late', 'msg-early']) - expect((await readParityRow(db, owner))?.contentWatermarkAt).toBe( - midUpdateAt.toISOString(), - ) - } finally { - vi.useRealTimers() - } -}) - -test('mailbox parity content replay resumes durable upper and cursor across ticks', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'f'.repeat(64) - const baseline = new Date('2026-08-01T10:00:00.000Z') - vi.setSystemTime(baseline) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - const messageCount = mailboxParityContentPageSize + 2 - for (let index = 0; index < messageCount; index += 1) { - await insertMessage({ - db, - userId: owner, - id: `msg-${String(index).padStart(2, '0')}`, - createdAt: `2026-07-01T00:00:${String(index).padStart(2, '0')}.000Z`, - }) - } - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: messageCount, - attachments: 0, - deliveryEvents: 0, - }) - const env = parityEnv(db) - await reconcileMailboxParity({ env, now: baseline, batchSize: 1 }) - - const updateAt = '2026-08-01T10:30:00.000Z' - await db - .prepare(`UPDATE email_messages SET updated_at = ? WHERE user_id = ?`) - .bind(updateAt, owner) - .run() - - const replayNow = new Date('2026-08-01T11:00:00.000Z') - vi.setSystemTime(replayNow) - let contentCalls = 0 - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => { - contentCalls += 1 - if (contentCalls === mailboxParityContentPageSize) { - // Exhaust budget mid-window by freezing time past deadline. - vi.setSystemTime(new Date(replayNow.getTime() + 20_000)) - } - return successGraph(input.messageId) - }, - ) - await expect( - reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), - ).resolves.toMatchObject({ - compared: 0, - failed: 0, - backfilled: mailboxParityContentPageSize, - }) - const mid = await readParityRow(db, owner) - expect(mid).toMatchObject({ - contentWatermarkAt: baseline.toISOString(), - contentReplayUpperAt: replayNow.toISOString(), - contentReplayCursorId: `msg-${String(mailboxParityContentPageSize - 1).padStart(2, '0')}`, - matchingSince: null, - }) - - vi.setSystemTime(replayNow) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxMessageGraphFromD1.mockClear() - await expect( - reconcileMailboxParity({ env, now: replayNow, batchSize: 1 }), - ).resolves.toMatchObject({ - compared: 1, - matched: 1, - backfilled: 2, - }) - expect( - mocks.mirrorMailboxMessageGraphFromD1.mock.calls.map( - (call) => call[0].messageId as string, - ), - ).toEqual([ - `msg-${String(mailboxParityContentPageSize).padStart(2, '0')}`, - `msg-${String(mailboxParityContentPageSize + 1).padStart(2, '0')}`, - ]) - const done = await readParityRow(db, owner) - expect(done).toMatchObject({ - contentWatermarkAt: replayNow.toISOString(), - contentReplayUpperAt: null, - contentReplayCursorId: null, - // Soak was cleared on the incomplete tick; exact compare starts a new window. - matchingSince: replayNow.toISOString(), - }) - } finally { - vi.useRealTimers() - } -}) - -test('mailbox parity missing and unconfigured do not advance creation cursors', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'g'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertMessage({ - db, - userId: owner, - id: 'msg-2', - createdAt: '2026-07-01T00:00:01.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-1', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:02.000Z', - }) - - mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValueOnce({ - messageId: 'msg-1', - message: { status: 'missing' as const }, - events: [], - eventsTruncated: false, - }) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 1, - }) - const env = parityEnv(db) - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ failed: 1, backfilled: 0, compared: 0 }) - expect(await readParityRow(db, owner)).toMatchObject({ - messageCursorId: null, - messagesCompletedAt: null, - contentWatermarkAt: now.toISOString(), - lastError: 'message backfill message graph missing', - }) - - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValueOnce({ - status: 'skipped', - reason: 'mailbox-unconfigured', - }) - await expect( - reconcileMailboxParity({ - env, - now: new Date('2026-08-01T11:00:00.000Z'), - batchSize: 1, - }), - ).resolves.toMatchObject({ failed: 1, compared: 0 }) - const afterUnconfigured = await readParityRow(db, owner) - expect(afterUnconfigured).toMatchObject({ - messagesCompletedAt: expect.any(String), - eventCursorId: null, - eventsCompletedAt: null, - lastError: 'event backfill event skipped:mailbox-unconfigured', - }) -}) - -test('mailbox parity eventsTruncated still completes via full event phase', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'h'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - for (let index = 0; index < 3; index += 1) { - await insertEvent({ - db, - userId: owner, - id: `evt-${index}`, - messageId: 'msg-1', - createdAt: `2026-07-01T00:01:0${index}.000Z`, - }) - } - mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue( - successGraph('msg-1', true), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 3, - }) - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 4, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledTimes(3) -}) - -test('mailbox parity event missing advances cursor while unconfigured does not', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'j'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-gone', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:01.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-ok', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:02.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1 - .mockResolvedValueOnce({ status: 'missing' }) - .mockResolvedValueOnce({ status: 'mirrored' }) - mockDoCounts({ - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 2, - }) - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toMatchObject({ matched: 1, failed: 0 }) - expect(await readParityRow(db, owner)).toMatchObject({ - eventCursorId: 'evt-ok', - eventsCompletedAt: expect.any(String), - }) -}) - -test('mailbox parity deletion race purges Mailbox and skips parity writes', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'i'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - const purge = vi.fn(async () => ({ ok: true as const })) - mocks.mailboxRpc.mockImplementation(() => ({ - countMailbox: vi.fn(async () => ({ - threads: 0, - messages: 0, - attachments: 0, - deliveryEvents: 0, - })), - purge, - })) - mocks.awaitMailboxMirrorRpc.mockImplementation( - async (promise: Promise) => { - try { - return { ok: true as const, value: await promise } - } catch { - return { ok: false as const, result: { status: 'timeout' as const } } - } - }, - ) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { - await db - .prepare(`UPDATE users SET deleting_at = ? WHERE stable_user_id = ?`) - .bind(now.toISOString(), owner) - .run() - return successGraph('msg-1') - }) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - - await expect( - reconcileMailboxParity({ - env: parityEnv(db), - now, - batchSize: 1, - }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 0, - matched: 0, - mismatched: 0, - failed: 0, - }) - expect(purge).toHaveBeenCalledTimes(1) - const row = await readParityRow(db, owner) - // Baseline may have been written before deletion; no completion/compare state. - expect(row?.messagesCompletedAt).toBeNull() - expect(row?.matchingSince).toBeNull() - expect(row?.eventsCompletedAt).toBeNull() -}) - -test('tracked empty user discovers DO-only leftover, purges, then reaches zero parity', async () => { - consoleWarn.mockImplementation(() => {}) - const { db } = await createParityDb() - const owner = 'k'.repeat(64) - const neverTracked = 'l'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - await insertUser({ db, userId: owner, checkedAt: null }) - await insertUser({ db, userId: neverTracked, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - await insertEvent({ - db, - userId: owner, - id: 'evt-1', - messageId: 'msg-1', - createdAt: '2026-07-01T00:00:01.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - const purge = vi.fn(async () => ({ ok: true as const })) - mockDoCounts( - { - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 1, - }, - { purge }, - ) - const env = parityEnv(db) - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ matched: 1, compared: 1 }) - - // Simulate failed delete mirror: D1 cleared, DO still holds message/event. - await db - .prepare(`DELETE FROM email_delivery_events WHERE user_id = ?`) - .bind(owner) - .run() - await db - .prepare(`DELETE FROM email_messages WHERE user_id = ?`) - .bind(owner) - .run() - - expect( - (await listUsersForMailboxParity({ db, limit: 10 })).map( - (row) => row.userId, - ), - ).toEqual([owner]) - expect( - (await listUsersForMailboxParity({ db, limit: 10 })).map( - (row) => row.userId, - ), - ).not.toContain(neverTracked) - - mockDoCounts( - { - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 1, - }, - { purge }, - ) - const mismatchNow = new Date('2026-08-01T11:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: mismatchNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 0, - mismatched: 1, - failed: 0, - }) - expect(purge).toHaveBeenCalledTimes(1) - const afterPurge = await readParityRow(db, owner) - expect(afterPurge).toMatchObject({ - matchingSince: null, - mismatchCount: 1, - contentWatermarkAt: null, - messagesCompletedAt: null, - eventsCompletedAt: null, - messageCursorId: null, - eventCursorId: null, - lastError: null, - }) - - mockDoCounts({ - threads: 0, - messages: 0, - attachments: 0, - deliveryEvents: 0, - }) - const zeroNow = new Date('2026-08-01T12:00:00.000Z') - await expect( - reconcileMailboxParity({ env, now: zeroNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 1, - mismatched: 0, - failed: 0, - }) - const afterZero = await readParityRow(db, owner) - expect(afterZero).toMatchObject({ - matchingSince: zeroNow.toISOString(), - mismatchCount: 0, - contentWatermarkAt: expect.any(String), - }) - if (!afterZero?.contentWatermarkAt) { - throw new Error('Expected content watermark after zero-count parity.') - } - expect(afterZero.contentWatermarkAt >= zeroNow.toISOString()).toBe(true) -}) - -test('mismatch purge failure clears soak and does not start rebuild reset', async () => { - consoleWarn.mockImplementation(() => {}) - vi.useFakeTimers() - const { db } = await createParityDb() - const owner = 'm'.repeat(64) - const now = new Date('2026-08-01T10:00:00.000Z') - vi.setSystemTime(now) - - try { - await insertUser({ db, userId: owner, checkedAt: null }) - await insertMessage({ - db, - userId: owner, - id: 'msg-1', - createdAt: '2026-07-01T00:00:00.000Z', - }) - mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( - async (input: { messageId: string }) => successGraph(input.messageId), - ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) - mockDoCounts({ - threads: 0, - messages: 1, - attachments: 0, - deliveryEvents: 0, - }) - const env = parityEnv(db) - await expect( - reconcileMailboxParity({ env, now, batchSize: 1 }), - ).resolves.toMatchObject({ matched: 1 }) - const afterMatch = await readParityRow(db, owner) - expect(afterMatch?.messagesCompletedAt).toEqual(expect.any(String)) - expect(afterMatch?.contentWatermarkAt).toBe(now.toISOString()) - - mocks.mailboxRpc.mockImplementation(() => ({ - countMailbox: vi.fn(async () => ({ - threads: 0, - messages: 2, - attachments: 0, - deliveryEvents: 0, - })), - purge: vi.fn(async () => ({ ok: true as const })), - })) - mocks.awaitMailboxMirrorRpc.mockImplementation( - async (promise: Promise) => { - const settled = await promise - if (settled && typeof settled === 'object' && 'messages' in settled) { - return { ok: true as const, value: settled } - } - return { ok: false as const, timedOut: true as const } - }, - ) - consoleWarn.mockClear() - const failNow = new Date('2026-08-01T11:00:00.000Z') - vi.setSystemTime(failNow) - await expect( - reconcileMailboxParity({ env, now: failNow, batchSize: 1 }), - ).resolves.toEqual({ - scanned: 1, - backfilled: 0, - compared: 1, - matched: 0, - mismatched: 1, - failed: 1, - }) - expect(consoleWarn).toHaveBeenCalledWith( - 'mailbox-parity-mismatch-purge-failed', - owner, - 'timeout', - 'mailbox metadata purge timeout', - ) - const afterFail = await readParityRow(db, owner) - // Empty content window may advance the watermark before compare; purge - // failure must not clear creation completion/cursors (rebuild not started). - expect(afterFail).toMatchObject({ - matchingSince: null, - mismatchCount: 1, - messagesCompletedAt: afterMatch?.messagesCompletedAt, - eventsCompletedAt: afterMatch?.eventsCompletedAt, - messageCursorId: afterMatch?.messageCursorId, - lastError: 'mailbox metadata purge timeout', - checkedAt: failNow.toISOString(), - }) - expect(afterFail?.contentWatermarkAt).not.toBeNull() - } finally { - vi.useRealTimers() - } -}) From c43af54bbe9b6c40a0d4f795c3817651b8b1dbe5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:42:49 +0000 Subject: [PATCH 18/62] test(email): exclude parity support from worker build Co-authored-by: Kent C. Dodds --- .../mailbox-reconcile-cursor-backfill-content.node.test.ts | 2 +- .../mailbox-reconcile-mismatch-deletion-error.node.test.ts | 2 +- ...e-test-support.ts => mailbox-reconcile.node.test-support.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/worker/src/email/{mailbox-reconcile-test-support.ts => mailbox-reconcile.node.test-support.ts} (100%) diff --git a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts index 9e509c5547..c41a0d8f43 100644 --- a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts @@ -13,7 +13,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile-test-support.ts' +} from './mailbox-reconcile.node.test-support.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' import { systemEmailOwnerId } from './email-owner.ts' diff --git a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts index 6b8f9bd795..67557aa658 100644 --- a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts @@ -10,7 +10,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile-test-support.ts' +} from './mailbox-reconcile.node.test-support.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' diff --git a/packages/worker/src/email/mailbox-reconcile-test-support.ts b/packages/worker/src/email/mailbox-reconcile.node.test-support.ts similarity index 100% rename from packages/worker/src/email/mailbox-reconcile-test-support.ts rename to packages/worker/src/email/mailbox-reconcile.node.test-support.ts From 669e93bcc71887dd728d7b636f2d2aa7f21fd6b8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 12:43:31 +0000 Subject: [PATCH 19/62] test(email): relocate parity test support Co-authored-by: Kent C. Dodds --- ...-reconcile-cursor-backfill-content.node.test.ts | 2 +- ...-reconcile-mismatch-deletion-error.node.test.ts | 2 +- .../mailbox-reconcile.ts} | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) rename packages/worker/src/{email/mailbox-reconcile.node.test-support.ts => test-support/mailbox-reconcile.ts} (92%) diff --git a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts index c41a0d8f43..f5188db190 100644 --- a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts @@ -13,7 +13,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile.node.test-support.ts' +} from '#worker/test-support/mailbox-reconcile.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' import { systemEmailOwnerId } from './email-owner.ts' diff --git a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts index 67557aa658..2f41201db7 100644 --- a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts @@ -10,7 +10,7 @@ import { readParityRow, reconcileMailboxParity, successGraph, -} from './mailbox-reconcile.node.test-support.ts' +} from '#worker/test-support/mailbox-reconcile.ts' import { expect, test, vi } from 'vitest' import { consoleWarn } from '#worker/test-support/console-spies.ts' diff --git a/packages/worker/src/email/mailbox-reconcile.node.test-support.ts b/packages/worker/src/test-support/mailbox-reconcile.ts similarity index 92% rename from packages/worker/src/email/mailbox-reconcile.node.test-support.ts rename to packages/worker/src/test-support/mailbox-reconcile.ts index 2baf8e1cff..35ce0801c6 100644 --- a/packages/worker/src/email/mailbox-reconcile.node.test-support.ts +++ b/packages/worker/src/test-support/mailbox-reconcile.ts @@ -2,8 +2,8 @@ import { DatabaseSync } from 'node:sqlite' import { vi } from 'vitest' import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' -import type * as MailboxMirrorModule from './mailbox-mirror.ts' -import { ensureEmailTestSchema } from './test-schema.ts' +import type * as MailboxMirrorModule from '#worker/email/mailbox-mirror.ts' +import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' const mocks = vi.hoisted(() => ({ mirrorMailboxMessageGraphFromD1: vi.fn(), @@ -13,12 +13,12 @@ const mocks = vi.hoisted(() => ({ mailboxRpc: vi.fn(), })) -vi.mock('./mailbox-live-mirror.ts', () => ({ +vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, })) -vi.mock('./mailbox-mirror.ts', async (importOriginal) => { +vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { const actual = await importOriginal() return { ...actual, @@ -26,11 +26,11 @@ vi.mock('./mailbox-mirror.ts', async (importOriginal) => { } }) -vi.mock('./mailbox-parity-events.ts', () => ({ +vi.mock('#worker/email/mailbox-parity-events.ts', () => ({ recordMailboxParityEvent: mocks.recordMailboxParityEvent, })) -vi.mock('./mailbox-client.ts', () => ({ +vi.mock('#worker/email/mailbox-client.ts', () => ({ mailboxRpc: mocks.mailboxRpc, })) @@ -45,7 +45,7 @@ export const { listEventBackfillPage, listUsersForMailboxParity, reconcileMailboxParity, -} = await import('./mailbox-reconcile.ts') +} = await import('#worker/email/mailbox-reconcile.ts') export async function createParityDb() { const sqlite = new DatabaseSync(':memory:') From 03aa7e9a5c8cc9f45cd5a53e68a108aba70a7467 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 13:06:03 +0000 Subject: [PATCH 20/62] fix(email): preserve parity test module exports Co-authored-by: Kent C. Dodds --- .../migrations/0125-mailbox-parity-state.sql | 2 +- .../worker/src/email/mailbox-parity-repo.ts | 2 +- .../worker/src/email/mailbox-reconcile.ts | 2 +- .../src/test-support/mailbox-reconcile.ts | 35 +++++++++++++------ tools/migration-ledger.json | 2 +- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/worker/migrations/0125-mailbox-parity-state.sql b/packages/worker/migrations/0125-mailbox-parity-state.sql index 937a5a76e2..f0c29743bd 100644 --- a/packages/worker/migrations/0125-mailbox-parity-state.sql +++ b/packages/worker/migrations/0125-mailbox-parity-state.sql @@ -1,4 +1,4 @@ --- Account-owned Mailbox parity / backfill state for the hourly reconcile lane. +-- Account-owned Mailbox parity / backfill state for the five-minute reconcile lane. -- Low-write columns live on users (not per-message). Creation cursors are -- keyset (created_at, id) for messages and ALL owner delivery events so backfill -- can resume across ticks without DO enumeration. The event phase rescans every diff --git a/packages/worker/src/email/mailbox-parity-repo.ts b/packages/worker/src/email/mailbox-parity-repo.ts index 2ee1a86e06..fe1f767cba 100644 --- a/packages/worker/src/email/mailbox-parity-repo.ts +++ b/packages/worker/src/email/mailbox-parity-repo.ts @@ -2,7 +2,7 @@ import { systemEmailOwnerId } from './email-owner.ts' import { type MailboxCountResult } from './mailbox-types.ts' /** - * D1 loaders/persisters for the hourly Mailbox parity reconcile lane. + * D1 loaders/persisters for the five-minute Mailbox parity reconcile lane. * Creation backfill uses (created_at, id) keysets over messages and all owner * delivery events; content replay uses a durable (watermark, upper] window with * an (updated_at, id) cursor. diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index b2b1dec2dd..d2ee662310 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -33,7 +33,7 @@ import { import { type MailboxCountResult } from './mailbox-types.ts' /** - * Bounded hourly Mailbox backfill + count-parity reconciler. + * Bounded five-minute Mailbox backfill + count-parity reconciler. * * Discovers a small oldest-first page of non-deleting D1 mail owners (no DO * enumeration), mirrors message graphs then every owner delivery event through diff --git a/packages/worker/src/test-support/mailbox-reconcile.ts b/packages/worker/src/test-support/mailbox-reconcile.ts index 35ce0801c6..0388df44bd 100644 --- a/packages/worker/src/test-support/mailbox-reconcile.ts +++ b/packages/worker/src/test-support/mailbox-reconcile.ts @@ -2,7 +2,10 @@ import { DatabaseSync } from 'node:sqlite' import { vi } from 'vitest' import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import type * as MailboxClientModule from '#worker/email/mailbox-client.ts' +import type * as MailboxLiveMirrorModule from '#worker/email/mailbox-live-mirror.ts' import type * as MailboxMirrorModule from '#worker/email/mailbox-mirror.ts' +import type * as MailboxParityEventsModule from '#worker/email/mailbox-parity-events.ts' import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' const mocks = vi.hoisted(() => ({ @@ -13,10 +16,14 @@ const mocks = vi.hoisted(() => ({ mailboxRpc: vi.fn(), })) -vi.mock('#worker/email/mailbox-live-mirror.ts', () => ({ - mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, - mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, -})) +vi.mock('#worker/email/mailbox-live-mirror.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, + mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, + } +}) vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { const actual = await importOriginal() @@ -26,13 +33,21 @@ vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { } }) -vi.mock('#worker/email/mailbox-parity-events.ts', () => ({ - recordMailboxParityEvent: mocks.recordMailboxParityEvent, -})) +vi.mock('#worker/email/mailbox-parity-events.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + recordMailboxParityEvent: mocks.recordMailboxParityEvent, + } +}) -vi.mock('#worker/email/mailbox-client.ts', () => ({ - mailboxRpc: mocks.mailboxRpc, -})) +vi.mock('#worker/email/mailbox-client.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + mailboxRpc: mocks.mailboxRpc, + } +}) export type MailboxReconcileTestMocks = typeof mocks diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 7cd41d25a2..070dc3f59f 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -514,7 +514,7 @@ }, { "filename": "0125-mailbox-parity-state.sql", - "sha256": "a0cfd916aff30eecf74d315088c131a8fd9f800f44aa8ed4406a806cfa1f1cff" + "sha256": "6d2d54b8d8a2cc7399c4f0dbb8069cfa5d6686958d8fef1a3b874c472a564754" } ] } From 8ae4d48b7cd03f9aa3e2663bb24766402c468135 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 13:25:01 +0000 Subject: [PATCH 21/62] feat(email): dual-write finalized inbound Mailbox state Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 185 ++++-- .../inbound-mailbox-mirror.workers.test.ts | 626 ++++++++++++++++++ packages/worker/src/email/inbound.ts | 116 +++- .../repo-delete-mailbox-mirror.node.test.ts | 236 +++++++ packages/worker/src/email/repo.ts | 32 + .../service-attachment-cleanup.node.test.ts | 139 ++++ packages/worker/src/email/service.ts | 10 + 7 files changed, 1273 insertions(+), 71 deletions(-) create mode 100644 packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts create mode 100644 packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts create mode 100644 packages/worker/src/email/service-attachment-cleanup.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 6b0231359d..609466919e 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -219,11 +219,12 @@ Durable Object export behavior: delivery events) through the account-export `mailbox` section (`exportMailbox` RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires - non-inbound live dual-write for outbound terminals, provider delivery-queue - graph repair, and user classification - (`service.ts#setEmailMessageClassification`) without changing D1 authority — - D1 `email_*` rows remain the live source of truth and are still exported in - the `d1` section. See [Mailbox](#durable-objects-mailbox). + live dual-write for outbound terminals, provider delivery-queue graph repair, + user classification (`service.ts#setEmailMessageClassification`), high-risk + inbound terminal paths, and `deleteEmailMessageById` metadata delete without + changing D1 authority — D1 `email_*` rows remain the live source of truth and + are still exported in the `d1` section. See + [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -516,11 +517,16 @@ rows only. re-parses the resolved MIME from that blob. - Message deletes always delete the deterministic `emailRawMimeKey(userId, messageId)` from R2 (production writers always store - that canonical key): `deleteEmailMessageById` (best-effort after the D1 row - delete), user email retention and system-email retention (strict: blob delete - before row delete; failed blob deletes skip the row for retry), and account - deletion (strict before atomic D1 finalization; a failed blob delete preserves - every message row for retry). + that canonical key). `deleteEmailMessageById` runs an atomic D1 batch + (attachments, then message), then best-effort + `mirrorMailboxDeleteMessageMetadata` when `env` is supplied (metadata-only; + nulls delivery-event `message_id`, never deletes R2 or empty threads — thread + cleanup stays deferred), then best-effort R2 blob deletes in the same order as + before expand. Mirror failures/timeouts never affect return/throw; optional + `waitUntil` attaches the mirror without blocking R2. User email retention and + system-email retention stay strict: blob delete before row delete; failed blob + deletes skip the row for retry. Account deletion stays strict before atomic D1 + finalization; a failed blob delete preserves every message row for retry. - Bucket names: `kody-email-blobs` (production), per-preview `{worker}-email-blobs` buckets created and cleaned up by `tools/ci/preview-resources.ts`, and the test env reuses the preview-style @@ -704,10 +710,12 @@ preserves existing attachment rows; an explicit `attachments: []` clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment `storageKey` values against the canonical builders for that `ownerId`. -**Partial mutation RPCs** (library-only in the non-inbound phase-2 slice; -owner-bound, monotonic on `updatedAt`, no R2 or retention-alarm side effects — -unlike full snapshot mirrors, these do not mark retention dirty or reschedule -alarms): +**Partial mutation RPCs** (owner-bound, monotonic on `updatedAt`, no R2 or +retention-alarm side effects — unlike full snapshot mirrors, these do not mark +retention dirty or reschedule alarms). `deleteMessageMetadata` is live on the +`deleteEmailMessageById` path when `env` is supplied; touch/update/classify and +the other delete RPCs remain library-only on live paths that prefer full graph +repair: - `touchThread` — advance `last_message_at` / `updated_at` without a full snapshot; `last_message_at` never moves backward @@ -768,8 +776,10 @@ never propagate into D1 commit paths. Helpers cover full snapshots (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). Partial -mutation helpers remain library-only; live paths prefer the graph orchestrator -below. D1 remains sole authority for all live mail read/write paths. +mutation helpers are library-only except `mirrorMailboxDeleteMessageMetadata` on +the `deleteEmailMessageById` path; other live paths prefer the graph +orchestrator below. D1 remains sole authority for all live mail read/write +paths. **Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 message graph (optional caller thread, message, attachments, then delivery @@ -784,8 +794,7 @@ oldest overflow — with a stable warning (`mailbox-live-mirror-events-truncated one `mailbox_mirror:upsert_delivery_event_batch` telemetry outcome (timeout/error apply uniformly to per-event summary entries). Each graph attempt emits at most two Analytics Engine writes (1 message outcome + 1 batch outcome). -Never throws; returns a bounded summary. **Live callers (non-inbound phase-2 -slice):** +Never throws; returns a bounded summary. **Live callers:** - **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full @@ -803,10 +812,28 @@ slice):** transport handlers (`account-email.ts`, `email-message-classify.ts`) delegate here for the D1 mutation + full graph mirror invariant (mirror only after a successful D1 update; failures never change the mutation response). - -Inbound commit/touch/classify paths, retention deletes, and partial mutation -mirrors are **not** wired in this slice. Scheduled parity reconcile **is** wired -(see below); read cutover is prepared but not flipped. +- **Inbound terminals** (`inbound.ts`) — high-risk user-mail dual-write with + strict ordering: **no Mailbox RPC before** durable D1/R2 message + attachment + storage and `received` finalization win. The winner schedules full graph + repair via `ctx.waitUntil` (`scheduleInboundMailboxMessageGraphMirror`). + **Already-received** Email Routing retries (delivery ledger + `state === 'received'` with an existing message row) idempotently repair the + graph and re-run effect reconciliation without a second charge. **Rejected + terminals** (post-claim parse failure or replay of a claimed `rejected` + delivery) mirror the delivery event only via + `scheduleInboundMailboxDeliveryEventMirror` + (`mirrorMailboxDeliveryEventFromD1`). After successful + `processInboundDeliveryEffects`, the effects chain re-mirrors the updated + delivery event (usage/subscription fields). Mirror failures, timeouts, and + hangs never affect SMTP reject/refund/retry/charge semantics. **Pre-claim + bounded rejection rows** (`recordBoundedEmailRejectionEvent` for verification, + suspension, sender-policy, size, entitlement, and system-limit gates before + delivery claim/charge) stay **D1-only on the live path** — the every-5-minute + `mailbox_parity` lane backfills them. **`system:email` stays excluded** (no + per-user Mailbox object). Retention sweeper deletes and other bulk + metadata-delete mirrors are **still not wired** on live paths. Scheduled + parity reconcile **is** wired (see below); read cutover is prepared but not + flipped. **Scheduled parity reconcile (`mailbox_parity`)** runs as its own queue-isolated scheduled lane on **every** five-minute Worker cron tick @@ -914,27 +941,32 @@ mail content. Account export pages Mailbox state through the `mailbox` section ### Expand/contract phases -This is an expand/contract migration. **Phase 2 is partially wired (non-inbound -slice):** live dual-write covers outbound terminal +This is an expand/contract migration. **Phase 2 live paths are wired (terminal +inbound + parity):** live dual-write covers outbound terminal message/thread/attachment/event graphs, provider delivery-queue graph repair -(`recorded` / `duplicate` / `stale` with a message, via `waitUntil`), and user +(`recorded` / `duplicate` / `stale` with a message, via `waitUntil`), user classification (full graph repair after D1 update via -`service.ts#setEmailMessageClassification`). Mirror helpers emit automatic -namespaced outcome telemetry (`missing` and `timeout` included). Message and -batch event repair each use one 1s-bounded RPC; each graph attempt emits at most -two AE writes; event repair batches up to 100 events with explicit truncation -(newest retained, chronological insertion restored, stable truncation warning) -and one batch telemetry outcome. **`system:email` stays in D1 only.** The +`service.ts#setEmailMessageClassification`), high-risk inbound terminal paths +(received graph + rejected delivery-event mirror + post-effects event re-mirror; +`waitUntil`; no Mailbox before D1/R2 finalization; pre-claim bounded rejections +parity-lane only; `system:email` excluded), and `deleteEmailMessageById` +metadata delete (mirror only after the D1 batch; R2 unchanged; thread cleanup +deferred). Mirror helpers emit automatic namespaced outcome telemetry (`missing` +and `timeout` included). Message and batch event repair each use one 1s-bounded +RPC; each graph attempt emits at most two AE writes; event repair batches up to +100 events with explicit truncation (newest retained, chronological insertion +restored, stable truncation warning) and one batch telemetry outcome. The every-5-minute queue-isolated `mailbox_parity` lane backfills all owner messages -and delivery events, durable content-watermark replays, compares owner-scoped D1 -vs Mailbox counts, persists soak state on `users` (migration `0125`), and -re-purges the DO when account deletion races the lane. Inbound lifecycle -dual-write, retention/metadata deletes, and live read cutover are **still not -wired**. Live email read/write authority is unchanged — D1 `email_*` tables -remain authoritative for all live paths. D1 email rows and existing R2 inventory -deletion stay authoritative during expand. **Phase 2 completes only when every -user-mail mutation that writes D1 also writes the DO** — inbound paths and -delete mirrors remain pending. +and delivery events (including pre-claim bounded rejection rows), durable +content-watermark replays, compares owner-scoped D1 vs Mailbox counts, persists +soak state on `users` (migration `0125`), and re-purges the DO when account +deletion races the lane. Retention sweeper metadata-delete mirrors and live read +cutover are **still not wired**. Live email read/write authority is unchanged — +D1 `email_*` tables remain authoritative for all live paths. D1 email rows and +existing R2 inventory deletion stay authoritative during expand. **Phase 2 +contract completion** (every user-mail D1 mutation also writes the DO on live +paths, including retention deletes) remains pending; terminal inbound + parity +cover the high-risk live surface today. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -951,16 +983,21 @@ delete mirrors remain pending. `deleteDeliveryEvent`, `deleteThreadIfEmpty`, `upsertDeliveryEvents`), and `mailbox-parity-events.ts`, `mailbox-parity-repo.ts`, `mailbox-parity-phases.ts`, and `mailbox-reconcile.ts`. **Live today - (partial):** outbound terminals, provider delivery-queue outcomes with a - message (`recorded`, `duplicate`, `stale`; `waitUntil`), and user - classification (`service.ts#setEmailMessageClassification`) call - `mirrorMailboxMessageGraphFromD1`; mirror helpers record `mailbox_mirror:*` - outcomes automatically (graph repair uses message RPC + one batch event RPC; - at most two AE writes per attempt). The every-5-minute `mailbox_parity` - scheduled lane backfills all owner messages and delivery events, durable - content-watermark replays, count compares, and soak tracking on `users` - (migration `0125`). **Still pending for phase-2 completion:** inbound - commit/touch/classify dual-write and retention/metadata-delete mirrors. + (terminal inbound + parity):** outbound terminals, provider delivery-queue + outcomes with a message (`recorded`, `duplicate`, `stale`; `waitUntil`), user + classification (`service.ts#setEmailMessageClassification`), inbound terminal + paths in `inbound.ts` (received graph via `waitUntil` only after D1/R2 + + finalization; rejected delivery-event mirror; post-effects event re-mirror; + already-received idempotent repair; pre-claim bounded rejections parity-lane + only; `system:email` excluded), and `deleteEmailMessageById` metadata delete + (mirror after D1 batch only) call the live mirror helpers; graph repair uses + message RPC + one batch event RPC (at most two AE writes per attempt). The + every-5-minute `mailbox_parity` scheduled lane backfills all owner messages + and delivery events, durable content-watermark replays, count compares, and + soak tracking on `users` (migration `0125`). **Still pending for phase-2 + contract completion:** retention sweeper metadata-delete mirrors. D1 + authority and read cutover are unchanged (prepared adapter exists; not + wired). 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified (`mailbox_parity_matching_since` ≥ 24h continuous exact counts, fresh @@ -997,29 +1034,43 @@ below. owning user. Contextless provider-id reverse lookups must not require enumerating per-user Mailbox objects. -### Inbound durability boundary (eventual DO phase) +### Inbound durability boundary (D1-authoritative dual-write) Today's D1-authoritative inbound commit boundary is documented under -[R2 (`EMAIL_BLOBS`)](#r2-community_assets-email_blobs): thread prework + R2 put - -- D1 message/attachment rows are pre-commit; `touchEmailThread` / `received` - delivery-event writes are post-commit best-effort; ambiguous attachment-insert - failures acknowledge rather than risk duplicates. +[R2 (`EMAIL_BLOBS`)](#r2-community_assets-email_blobs): thread prework, R2 put, +D1 message/attachment rows are pre-commit; `touchEmailThread` / `received` +delivery-event writes are post-commit best-effort; ambiguous attachment-insert +failures acknowledge rather than risk duplicates. + +**Mailbox dual-write ordering (phase-2 live, D1 still authoritative):** + +- **Pre-commit (no Mailbox):** thread prework, R2 raw-MIME put, D1 + message/attachment storage, and inbound delivery finalization to `received`. +- **Post-commit (best effort, `waitUntil`):** full message graph mirror + (`mirrorMailboxMessageGraphFromD1`) only after the durable commit + + finalization win; already-received retries repair the graph idempotently + without a second charge. Rejected **post-claim** terminals mirror the delivery + event only (`mirrorMailboxDeliveryEventFromD1`). After successful effect + dispatch, re-mirror the updated delivery event. Failures/timeouts never affect + reject, refund, retry, or charge semantics. +- **Pre-claim bounded rejections** (`recordBoundedEmailRejectionEvent` before + delivery claim/charge) write D1 audit rows only on the live path; the + `mailbox_parity` lane backfills them. **`system:email` is excluded** from all + Mailbox mirrors. +- **Ambiguity:** if attachment commit fails but message cleanup (or a residual + probe) cannot prove the pre-commit state, acknowledge the already-created + message (logged, non-retry) rather than risking a duplicate on Email Routing + retry. `deleteEmailMessageById` mirrors metadata only after the D1 batch; R2 + delete behavior is unchanged; empty-thread cleanup stays deferred. -When Mailbox becomes authoritative, the same shape applies with the DO as the -metadata store: +When Mailbox becomes read-authoritative (phase 3+), the same shape applies with +the DO as the metadata store: - **Pre-commit:** thread prework + R2 raw-MIME put + atomic Mailbox message/attachment commit. - **Post-commit (best effort):** thread touch and delivery-event writes — log failures without throwing (retry would duplicate mail). -- **Ambiguity:** if attachment commit fails but message cleanup (or a residual - probe) cannot prove the pre-commit state, acknowledge the already-created - message (logged, non-retry) rather than risking a duplicate on Email Routing - retry. - -Phase 1 scaffolding and the non-inbound phase-2 live slice do not change that -live D1 inbound boundary; inbound dual-write remains pending phase-2 completion. +- **Ambiguity:** same acknowledge-over-retry rule as today. ### Package state model @@ -1140,8 +1191,8 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` reconciliation alarms) - `MAILBOX` (Durable Objects; per-user email metadata — see [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export - consumption; phase 2 wires non-inbound live dual-write without changing D1 - authority) + consumption; phase 2 wires terminal inbound + outbound live dual-write and + `deleteEmailMessageById` metadata delete without changing D1 authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) diff --git a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts new file mode 100644 index 0000000000..6cfb4c3ec1 --- /dev/null +++ b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts @@ -0,0 +1,626 @@ +import { env } from 'cloudflare:workers' +import { expect, test, vi } from 'vitest' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' +import { buildInboundDelivery } from './inbound-delivery.ts' +import { handleInboundEmail } from './inbound.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import { rpcFor } from './mailbox-test-helpers.ts' +import * as parser from './parser.ts' +import { listEmailMessages } from './repo.ts' +import { RetryableInboundStorageError } from './service.ts' +import { createForwardableEmailMessage } from './test-fixtures.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const platformBaseUrl = 'https://kody.example.com' +const platformDomain = 'inbox.kody.example.com' + +function createInboundEnv() { + return { ...env, APP_BASE_URL: platformBaseUrl } +} + +async function seedAccount(input: { + db: D1Database + email: string + username: string + emailVerifiedAt?: string | null +}) { + const stableUserId = await createStableUserIdFromEmail(input.email) + await input.db + .prepare( + `INSERT INTO users (username, email, password_hash, email_verified_at, stable_user_id, plan) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(email) DO UPDATE SET + username = excluded.username, + email_verified_at = excluded.email_verified_at, + stable_user_id = COALESCE(users.stable_user_id, excluded.stable_user_id), + plan = excluded.plan, + updated_at = CURRENT_TIMESTAMP`, + ) + .bind( + input.username, + input.email, + 'test-password-hash', + input.emailVerifiedAt === undefined + ? new Date().toISOString() + : input.emailVerifiedAt, + stableUserId, + 'max', + ) + .run() +} + +async function seedVerifiedAccount(input: { + db: D1Database + email: string + username: string +}) { + await seedAccount(input) +} + +async function readUserDailyReceiveCount(userId: string) { + const result = await userMeterRpc({ env, userId }).read({ + resource: 'email_receives_per_day', + day: new Date().toISOString().slice(0, 10), + }) + return result.outcome === 'ready' ? result.count : 0 +} + +function createFailingEmailBlobs() { + return new Proxy(env.EMAIL_BLOBS, { + get(target, property, receiver) { + if (property === 'put') { + return async () => { + throw new Error('simulated R2 outage') + } + } + const value = Reflect.get(target, property, receiver) + return typeof value === 'function' ? value.bind(target) : value + }, + }) +} + +const inboundMailboxMirrorTimeoutMs = 30_000 + +/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ +function createInboundMailboxStubEnv(input: { + mode: 'throw' | 'hang' + base?: typeof env +}) { + const base = input.base ?? env + const hang = () => new Promise(() => {}) + const fail = async () => { + throw new Error('simulated mailbox failure') + } + const method = input.mode === 'hang' ? hang : fail + const stub = { + mirrorMessage: method, + upsertDeliveryEvent: method, + upsertDeliveryEvents: method, + touchThread: method, + updateMessageDelivery: method, + setMessageClassification: method, + deleteMessageMetadata: method, + deleteDeliveryEvent: method, + deleteThreadIfEmpty: method, + } + return { + ...base, + APP_BASE_URL: platformBaseUrl, + MAILBOX: { + idFromName: (name: string) => base.MAILBOX.idFromName(name), + get: () => stub, + } as unknown as DurableObjectNamespace, + } +} + +function buildInboundAttachmentRaw(input: { + address: string + subject: string + messageIdHeader: string + body: string + attachmentText: string +}) { + return [ + 'From: Sender ', + `To: ${input.address}`, + `Subject: ${input.subject}`, + `Message-ID: <${input.messageIdHeader}>`, + 'Content-Type: multipart/mixed; boundary="mail-boundary"', + '', + '--mail-boundary', + 'Content-Type: text/plain; charset="utf-8"', + '', + input.body, + '--mail-boundary', + 'Content-Type: text/plain; name="note.txt"', + 'Content-Disposition: attachment; filename="note.txt"', + '', + input.attachmentText, + '--mail-boundary--', + ].join('\r\n') +} + +test( + 'inbound finalization mirrors message, thread, attachments, and received event into Mailbox', + async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + await ensureUsageRollupsTestSchema(env.APP_DB) + const username = `mbx-in-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-in-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const raw = buildInboundAttachmentRaw({ + address, + subject: 'Mailbox inbound mirror', + messageIdHeader: 'mailbox-inbound@example.net', + body: 'Mirrored inbound body.', + attachmentText: 'Attachment bytes', + }) + const message = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail(message, createInboundEnv()) + expect(message.rejectedReason).toBeNull() + + const [stored] = await listEmailMessages({ + db: env.APP_DB, + userId, + limit: 1, + }) + expect(stored).toBeDefined() + if (!stored) throw new Error('Expected stored inbound message') + expect(stored.threadId).toBeTruthy() + + const mirroredMessage = await mailbox.getMessage({ + messageId: stored.id, + }) + expect(mirroredMessage).toMatchObject({ + id: stored.id, + direction: 'inbound', + processingStatus: 'stored', + subject: 'Mailbox inbound mirror', + threadId: stored.threadId, + }) + expect( + await mailbox.getThread({ threadId: stored.threadId! }), + ).toMatchObject({ + id: stored.threadId, + subjectNormalized: 'mailbox inbound mirror', + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: stored.id }), + ).toEqual([ + expect.objectContaining({ + filename: 'note.txt', + contentType: 'text/plain', + storageKind: 'raw-mime', + }), + ]) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: stored.id, + limit: 10, + }) + expect(mirroredEvents.map((event) => event.eventType)).toEqual(['received']) + expect(mirroredEvents[0]).toMatchObject({ + id: expect.stringMatching(/^email-inbound-delivery:/), + provider: 'cloudflare-email-routing', + }) + expect(await readUserDailyReceiveCount(userId)).toBe(1) + }, + inboundMailboxMirrorTimeoutMs, +) + +test( + 'Email Routing retry creates one message and charge with idempotent Mailbox mirror', + async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const username = `mbx-retry-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-retry-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const raw = buildInboundAttachmentRaw({ + address, + subject: 'Idempotent inbound mirror', + messageIdHeader: 'mailbox-idempotent@example.net', + body: 'Same bytes twice.', + attachmentText: 'Same attachment', + }) + const first = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + const second = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail(first, createInboundEnv()) + await handleInboundEmail(second, createInboundEnv()) + expect(first.rejectedReason).toBeNull() + expect(second.rejectedReason).toBeNull() + + const messages = await listEmailMessages({ + db: env.APP_DB, + userId, + limit: 10, + }) + expect(messages).toHaveLength(1) + expect(await readUserDailyReceiveCount(userId)).toBe(1) + + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: messages[0]!.id, + limit: 10, + }) + expect(mirroredEvents).toHaveLength(1) + expect(mirroredEvents[0]?.eventType).toBe('received') + expect( + await mailbox.getMessage({ messageId: messages[0]!.id }), + ).toMatchObject({ + id: messages[0]!.id, + direction: 'inbound', + }) + }, + inboundMailboxMirrorTimeoutMs, +) + +test( + 'pre-commit inbound failure retries without Mailbox message and preserves one charge', + async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const username = `mbx-pre-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-pre-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Pre-commit mailbox guard', + 'Message-ID: ', + '', + 'Should not mirror.', + ].join('\r\n') + const failingEnv = { + ...createInboundEnv(), + EMAIL_BLOBS: createFailingEmailBlobs(), + } as Parameters[1] + + const first = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await expect(handleInboundEmail(first, failingEnv)).rejects.toBeInstanceOf( + RetryableInboundStorageError, + ) + expect(first.rejectedReason).toBeNull() + expect(await readUserDailyReceiveCount(userId)).toBe(1) + expect( + await listEmailMessages({ db: env.APP_DB, userId, limit: 10 }), + ).toEqual([]) + + const candidate = await buildInboundDelivery({ + userId, + inboxId: 'unused', + recipient: address, + envelopeFrom: 'sender@example.net', + rawMime: raw, + quotaDay: new Date().toISOString().slice(0, 10), + }) + expect( + await mailbox.getMessage({ messageId: candidate.messageId }), + ).toBeNull() + + const retry = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail(retry, createInboundEnv()) + expect(retry.rejectedReason).toBeNull() + expect(await readUserDailyReceiveCount(userId)).toBe(1) + const [stored] = await listEmailMessages({ + db: env.APP_DB, + userId, + limit: 1, + }) + expect(stored?.id).toBe(candidate.messageId) + expect( + await mailbox.getMessage({ messageId: candidate.messageId }), + ).toMatchObject({ + id: candidate.messageId, + direction: 'inbound', + }) + }, + inboundMailboxMirrorTimeoutMs, +) + +test( + 'inbound Mailbox mirror timeout and error are acknowledged without throw or reject', + async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-mirror-message-failed', + 'mailbox-mirror-delivery-event-failed', + 'mailbox-mirror-delivery-event-batch-failed', + 'mailbox-live-mirror-message-graph-failed', + 'mailbox-live-mirror-delivery-event-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + + { + const username = `mbx-err-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-err-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Mailbox throw ignored', + 'Message-ID: ', + '', + 'Body', + ].join('\r\n') + const message = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail( + message, + createInboundMailboxStubEnv({ mode: 'throw' }), + ) + expect(message.rejectedReason).toBeNull() + expect(await readUserDailyReceiveCount(userId)).toBe(1) + expect( + await listEmailMessages({ db: env.APP_DB, userId, limit: 1 }), + ).toHaveLength(1) + } + + { + const username = `mbx-hang-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-hang-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Mailbox hang ignored', + 'Message-ID: ', + '', + 'Body', + ].join('\r\n') + const message = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + const startedAt = Date.now() + await handleInboundEmail( + message, + createInboundMailboxStubEnv({ mode: 'hang' }), + ) + const elapsedMs = Date.now() - startedAt + expect(message.rejectedReason).toBeNull() + expect(await readUserDailyReceiveCount(userId)).toBe(1) + expect( + await listEmailMessages({ db: env.APP_DB, userId, limit: 1 }), + ).toHaveLength(1) + // Graph + post-effects event each race the mirror RPC bound. + expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 8) + } + }, + inboundMailboxMirrorTimeoutMs, +) + +test( + 'already-received inbound replay repairs Mailbox graph without a second charge', + async () => { + silenceIncidentalRuntimeWarnings([ + 'mailbox-mirror-message-failed', + 'mailbox-mirror-delivery-event-failed', + 'mailbox-mirror-delivery-event-batch-failed', + 'mailbox-live-mirror-message-graph-failed', + 'mailbox-live-mirror-delivery-event-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + const username = `mbx-replay-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-replay-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const raw = buildInboundAttachmentRaw({ + address, + subject: 'Replay repair', + messageIdHeader: 'mailbox-replay@example.net', + body: 'Stored before repair.', + attachmentText: 'Repair me', + }) + const first = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail( + first, + createInboundMailboxStubEnv({ mode: 'throw' }), + ) + expect(first.rejectedReason).toBeNull() + const [stored] = await listEmailMessages({ + db: env.APP_DB, + userId, + limit: 1, + }) + expect(stored).toBeDefined() + if (!stored) throw new Error('Expected stored inbound message') + expect(await mailbox.getMessage({ messageId: stored.id })).toBeNull() + expect(await readUserDailyReceiveCount(userId)).toBe(1) + + const replay = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail(replay, createInboundEnv()) + expect(replay.rejectedReason).toBeNull() + expect(await readUserDailyReceiveCount(userId)).toBe(1) + expect( + await listEmailMessages({ db: env.APP_DB, userId, limit: 10 }), + ).toHaveLength(1) + + expect(await mailbox.getMessage({ messageId: stored.id })).toMatchObject({ + id: stored.id, + direction: 'inbound', + subject: 'Replay repair', + threadId: stored.threadId, + }) + expect( + await mailbox.listAttachmentsForMessage({ messageId: stored.id }), + ).toEqual([ + expect.objectContaining({ + filename: 'note.txt', + storageKind: 'raw-mime', + }), + ]) + expect( + ( + await mailbox.listDeliveryEvents({ messageId: stored.id, limit: 10 }) + ).map((event) => event.eventType), + ).toEqual(['received']) + }, + inboundMailboxMirrorTimeoutMs, +) + +test( + 'terminal inbound rejection mirrors the delivery event into Mailbox', + async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const username = `mbx-rej-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-rej-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Parse reject mirror', + 'Message-ID: ', + '', + 'Body', + ].join('\r\n') + const parseSpy = vi + .spyOn(parser, 'parseForwardableEmailRawMime') + .mockRejectedValueOnce(new Error('simulated parse failure')) + try { + const message = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail(message, createInboundEnv()) + expect(message.rejectedReason).toBe('simulated parse failure') + expect(await readUserDailyReceiveCount(userId)).toBe(1) + expect( + await listEmailMessages({ db: env.APP_DB, userId, limit: 10 }), + ).toEqual([]) + + const delivery = await env.APP_DB.prepare( + `SELECT id, event_type FROM email_delivery_events + WHERE user_id = ? AND provider = 'cloudflare-email-routing' + AND event_type = 'rejected'`, + ) + .bind(userId) + .first<{ id: string; event_type: string }>() + expect(delivery?.event_type).toBe('rejected') + if (!delivery) throw new Error('Expected rejected delivery ledger.') + + const mirrored = await mailbox.listDeliveryEvents({ + messageId: null, + limit: 20, + }) + const rejectedEvent = mirrored.find((event) => event.id === delivery.id) + expect(rejectedEvent).toMatchObject({ + id: delivery.id, + eventType: 'rejected', + provider: 'cloudflare-email-routing', + messageId: null, + }) + + // Claimed-rejected replay re-mirrors the same event idempotently. + const replay = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail(replay, createInboundEnv()) + expect(replay.rejectedReason).toBe('simulated parse failure') + const afterReplay = await mailbox.listDeliveryEvents({ + messageId: null, + limit: 20, + }) + expect( + afterReplay.filter((event) => event.id === delivery.id), + ).toHaveLength(1) + expect(await readUserDailyReceiveCount(userId)).toBe(1) + } finally { + parseSpy.mockRestore() + } + }, + inboundMailboxMirrorTimeoutMs, +) diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index 8841678f42..b00aae8b49 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -51,6 +51,11 @@ import { recordEmailReportingEvent, type EmailReportingEnv, } from './reporting-events.ts' +import { + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, +} from './mailbox-live-mirror.ts' import { deleteEmptyEmailThreads, getEmailMessageById } from './repo.ts' import { recordBoundedEmailRejectionEvent, @@ -66,6 +71,59 @@ import { type SystemEmailLocal, } from './system-email.ts' +type InboundMailboxMirrorEnv = Pick< + Env, + 'APP_DB' | 'MAILBOX' | 'EMAIL_EVENTS' +> & + MailboxLiveMirrorEnv + +/** + * Best-effort D1 → Mailbox graph repair after durable inbound commit. + * Uses `ctx.waitUntil` when present; otherwise awaits the never-throw mirror. + * Must not run before D1/R2 storage + `markInboundDeliveryReceived`. + */ +async function scheduleInboundMailboxMessageGraphMirror(input: { + env: InboundMailboxMirrorEnv + userId: string + messageId: string + ctx?: ExecutionContext +}) { + const promise = mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + messageId: input.messageId, + }) + if (input.ctx) { + input.ctx.waitUntil(promise) + return + } + await promise +} + +/** + * Best-effort D1 → Mailbox delivery-event mirror (rejected / post-effects). + * Uses `ctx.waitUntil` when present; otherwise awaits the never-throw mirror. + */ +async function scheduleInboundMailboxDeliveryEventMirror(input: { + env: InboundMailboxMirrorEnv + userId: string + deliveryId: string + ctx?: ExecutionContext +}) { + const promise = mirrorMailboxDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + eventId: input.deliveryId, + }) + if (input.ctx) { + input.ctx.waitUntil(promise) + return + } + await promise +} + /** * Rejection audit writes are best-effort (the SMTP reject already happened), * but a failure must still be visible to operators — silently losing the @@ -158,7 +216,8 @@ async function cleanupInboundDurability(input: { } async function scheduleInboundDeliveryEffects(input: { - env: Parameters[0]['env'] + env: Parameters[0]['env'] & + InboundMailboxMirrorEnv userId: string deliveryId: string expectedFinalizationToken?: string @@ -169,6 +228,8 @@ async function scheduleInboundDeliveryEffects(input: { const waitUntil = input.ctx ? (promise: Promise) => input.ctx!.waitUntil(promise) : undefined + // On success, re-mirror the updated delivery event (usage/subscription + // effect fields). Failures keep the existing log + reconcile path only. const promise = processInboundDeliveryEffects({ env: input.env, userId: input.userId, @@ -176,6 +237,14 @@ async function scheduleInboundDeliveryEffects(input: { expectedFinalizationToken: input.expectedFinalizationToken, durationMs: input.durationMs, waitUntil, + }).then(async () => { + if (input.userId === systemEmailOwnerId) return + await mirrorMailboxDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + eventId: input.deliveryId, + }) }) if (input.ctx) { input.ctx.waitUntil( @@ -201,8 +270,11 @@ export async function handleInboundEmail( | 'USER_EMAIL_DOMAIN' | 'USAGE_EVENTS' | 'USER_METER' + | 'MAILBOX' + | 'EMAIL_EVENTS' > & - EmailReportingEnv, + EmailReportingEnv & + MailboxLiveMirrorEnv, ctx?: ExecutionContext, ) { const recipient = normalizeEmailAddress(message.to) @@ -593,6 +665,12 @@ export async function handleInboundEmail( message.setReject( claimedDelivery.rejectionReason ?? 'Failed to parse inbound email.', ) + await scheduleInboundMailboxDeliveryEventMirror({ + env, + userId, + deliveryId: claimedDelivery.deliveryId, + ctx, + }) return } if (claimedDelivery.state === 'received') { @@ -602,6 +680,12 @@ export async function handleInboundEmail( messageId: claimedDelivery.messageId, }) if (existing) { + await scheduleInboundMailboxMessageGraphMirror({ + env, + userId, + messageId: claimedDelivery.messageId, + ctx, + }) await scheduleInboundDeliveryEffects({ env, userId, @@ -629,6 +713,12 @@ export async function handleInboundEmail( }) if (!rejected) return await recordReceiveUsage({ outcome: 'error' }) + await scheduleInboundMailboxDeliveryEventMirror({ + env, + userId, + deliveryId: claimedDelivery.deliveryId, + ctx, + }) return } const storageClaim = await claimInboundDeliveryStorage({ @@ -638,7 +728,15 @@ export async function handleInboundEmail( usageStartedAt: new Date(receiveStartedAtMs).toISOString(), }) if (!storageClaim.claimed) { - if (storageClaim.delivery?.state === 'received') return + if (storageClaim.delivery?.state === 'received') { + await scheduleInboundMailboxMessageGraphMirror({ + env, + userId, + messageId: claimedDelivery.messageId, + ctx, + }) + return + } throw new RetryableInboundStorageError( 'Inbound delivery is already being stored; retry the stable delivery.', ) @@ -664,7 +762,14 @@ export async function handleInboundEmail( }) throw error } + // Mailbox dual-write only after durable D1/R2 commit + finalization win. if (!storedResult.wonFinalization) return + await scheduleInboundMailboxMessageGraphMirror({ + env, + userId, + messageId: storedResult.finalizedDelivery.messageId, + ctx, + }) await scheduleInboundDeliveryEffects({ env, userId, @@ -688,7 +793,10 @@ async function handleSystemInboundEmail(input: { | 'BUNDLE_ARTIFACTS_KV' | 'APP_BASE_URL' | 'USAGE_EVENTS' - > + | 'MAILBOX' + | 'EMAIL_EVENTS' + > & + MailboxLiveMirrorEnv recipient: string localPart: SystemEmailLocal systemDomain: string diff --git a/packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts b/packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts new file mode 100644 index 0000000000..804c2ed120 --- /dev/null +++ b/packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts @@ -0,0 +1,236 @@ +import { expect, test, vi } from 'vitest' +import { emailRawMimeKey } from './blob-keys.ts' + +const mocks = vi.hoisted(() => ({ + mirrorMailboxDeleteMessageMetadata: vi.fn(async () => ({ + status: 'mirrored' as const, + })), +})) + +vi.mock('./mailbox-mirror.ts', () => ({ + mirrorMailboxDeleteMessageMetadata: mocks.mirrorMailboxDeleteMessageMetadata, +})) + +const { deleteEmailMessageById } = await import('./repo.ts') + +function createDeleteHarness(input?: { + userId?: string | null + attachmentKeys?: Array + batchError?: Error +}) { + const userId = input?.userId === undefined ? 'user-1' : input.userId + const attachmentKeys = input?.attachmentKeys ?? ['attachments/user-1/a1'] + const callOrder: Array = [] + const deletedBlobKeys: Array = [] + + const selectUser = { + bind: vi.fn(() => ({ + first: vi.fn(async () => (userId == null ? null : { user_id: userId })), + })), + } + const selectAttachments = { + bind: vi.fn(() => ({ + all: vi.fn(async () => ({ + results: attachmentKeys.map((storage_key) => ({ storage_key })), + })), + })), + } + const deleteAttachments = { + bind: vi.fn(() => ({ kind: 'delete-attachments' })), + } + const deleteMessage = { + bind: vi.fn(() => ({ kind: 'delete-message' })), + } + + let prepareCalls = 0 + const db = { + prepare: vi.fn((sql: string) => { + prepareCalls += 1 + if (sql.includes('SELECT user_id')) return selectUser + if (sql.includes('SELECT storage_key')) return selectAttachments + if (sql.includes('DELETE FROM email_attachments')) + return deleteAttachments + if (sql.includes('DELETE FROM email_messages')) return deleteMessage + throw new Error(`unexpected prepare: ${sql}`) + }), + batch: vi.fn(async () => { + callOrder.push('d1-batch') + if (input?.batchError) throw input.batchError + return [] + }), + } as unknown as D1Database + + const blobs = { + delete: vi.fn(async (key: string) => { + callOrder.push(`r2:${key}`) + deletedBlobKeys.push(key) + }), + } as unknown as R2Bucket + + const env = { APP_DB: db } as unknown as Env + + return { + db, + blobs, + env, + callOrder, + deletedBlobKeys, + prepareCalls: () => prepareCalls, + messageId: 'message-1', + userId, + attachmentKeys, + } +} + +test('deleteEmailMessageById keeps D1-then-R2 order and mirrors only after D1', async () => { + const harness = createDeleteHarness() + mocks.mirrorMailboxDeleteMessageMetadata.mockImplementation(async () => { + harness.callOrder.push('mirror') + return { status: 'mirrored' } + }) + + await deleteEmailMessageById({ + db: harness.db, + blobs: harness.blobs, + messageId: harness.messageId, + env: harness.env, + }) + + expect(harness.callOrder[0]).toBe('d1-batch') + expect(harness.callOrder[1]).toBe('mirror') + expect(harness.callOrder.slice(2)).toEqual([ + `r2:${emailRawMimeKey('user-1', harness.messageId)}`, + 'r2:attachments/user-1/a1', + ]) + expect(mocks.mirrorMailboxDeleteMessageMetadata).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxDeleteMessageMetadata).toHaveBeenCalledWith({ + env: harness.env, + ownerId: 'user-1', + messageId: harness.messageId, + deletedAt: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T/), + }) + expect(harness.deletedBlobKeys).toEqual([ + emailRawMimeKey('user-1', harness.messageId), + 'attachments/user-1/a1', + ]) +}) + +test('deleteEmailMessageById without mirror env preserves D1/R2 behavior and skips DO delete', async () => { + const harness = createDeleteHarness() + + await deleteEmailMessageById({ + db: harness.db, + blobs: harness.blobs, + messageId: harness.messageId, + }) + + expect(harness.callOrder).toEqual([ + 'd1-batch', + `r2:${emailRawMimeKey('user-1', harness.messageId)}`, + 'r2:attachments/user-1/a1', + ]) + expect(mocks.mirrorMailboxDeleteMessageMetadata).not.toHaveBeenCalled() +}) + +test('deleteEmailMessageById skips mirror when the message row is absent', async () => { + const harness = createDeleteHarness({ userId: null, attachmentKeys: [] }) + + await deleteEmailMessageById({ + db: harness.db, + blobs: harness.blobs, + messageId: harness.messageId, + env: harness.env, + }) + + expect(harness.callOrder).toEqual(['d1-batch']) + expect(mocks.mirrorMailboxDeleteMessageMetadata).not.toHaveBeenCalled() + expect(harness.deletedBlobKeys).toEqual([]) +}) + +test('deleteEmailMessageById never throws when mirror reports failure or timeout', async () => { + const harness = createDeleteHarness({ attachmentKeys: [] }) + mocks.mirrorMailboxDeleteMessageMetadata + .mockResolvedValueOnce({ + status: 'error', + error: new Error('mailbox delete mirror failed'), + }) + .mockResolvedValueOnce({ status: 'timeout' }) + + await expect( + deleteEmailMessageById({ + db: harness.db, + blobs: harness.blobs, + messageId: harness.messageId, + env: harness.env, + }), + ).resolves.toBeUndefined() + + await expect( + deleteEmailMessageById({ + db: harness.db, + blobs: harness.blobs, + messageId: harness.messageId, + env: harness.env, + }), + ).resolves.toBeUndefined() + + expect(harness.deletedBlobKeys).toEqual([ + emailRawMimeKey('user-1', harness.messageId), + emailRawMimeKey('user-1', harness.messageId), + ]) +}) + +test('deleteEmailMessageById schedules mirror via waitUntil without awaiting before R2', async () => { + const harness = createDeleteHarness({ attachmentKeys: [] }) + let resolveMirror!: (value: { status: 'mirrored' }) => void + const mirrorPromise = new Promise<{ status: 'mirrored' }>((resolve) => { + resolveMirror = resolve + }) + mocks.mirrorMailboxDeleteMessageMetadata.mockReturnValueOnce(mirrorPromise) + + const waitUntilPromises: Array> = [] + const waitUntil = vi.fn((promise: Promise) => { + waitUntilPromises.push(promise) + }) + + const pending = deleteEmailMessageById({ + db: harness.db, + blobs: harness.blobs, + messageId: harness.messageId, + env: harness.env, + waitUntil, + }) + + await expect(pending).resolves.toBeUndefined() + expect(waitUntil).toHaveBeenCalledTimes(1) + expect(harness.callOrder).toEqual([ + 'd1-batch', + `r2:${emailRawMimeKey('user-1', harness.messageId)}`, + ]) + expect(harness.deletedBlobKeys).toEqual([ + emailRawMimeKey('user-1', harness.messageId), + ]) + + resolveMirror({ status: 'mirrored' }) + await Promise.all(waitUntilPromises) + expect(mocks.mirrorMailboxDeleteMessageMetadata).toHaveBeenCalledTimes(1) +}) + +test('deleteEmailMessageById does not mirror when D1 batch fails', async () => { + const harness = createDeleteHarness({ + batchError: new Error('simulated d1 batch failure'), + }) + + await expect( + deleteEmailMessageById({ + db: harness.db, + blobs: harness.blobs, + messageId: harness.messageId, + env: harness.env, + }), + ).rejects.toThrow('simulated d1 batch failure') + + expect(mocks.mirrorMailboxDeleteMessageMetadata).not.toHaveBeenCalled() + expect(harness.deletedBlobKeys).toEqual([]) + expect(harness.callOrder).toEqual(['d1-batch']) +}) diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 0327befafc..4072490f9a 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -4,6 +4,10 @@ import { utf8ByteLength, } from '@kody-internal/shared/backup-restore-safety.ts' import { parseJsonArray } from '@kody-internal/shared/json-parsing.ts' +import { + mirrorMailboxDeleteMessageMetadata, + type MailboxMirrorEnv, +} from './mailbox-mirror.ts' import { emailClassificationValues, type EmailAttachmentRecord, @@ -1060,6 +1064,15 @@ export async function searchEmailMessages(input: { return (result.results ?? []).map(mapMessageRow) } +/** + * Delete a message and its attachments from D1, then best-effort delete R2 + * blobs. When `env` is provided and the row existed, schedule a Mailbox + * metadata-delete mirror only after the authoritative D1 batch succeeds + * (`deletedAt` = canonical now). Mirror never affects return/throw; when + * `waitUntil` is supplied the mirror promise is attached there, otherwise it + * is awaited (bounded, never-throw). Absent rows skip DO delete. Thread + * cleanup stays deferred (RPC nulls delivery `message_id` only). + */ export async function deleteEmailMessageById(input: { db: D1Database /** @@ -1068,6 +1081,10 @@ export async function deleteEmailMessageById(input: { */ blobs: R2Bucket messageId: string + /** Optional Mailbox binding for post-D1 best-effort metadata delete. */ + env?: MailboxMirrorEnv + /** When set with `env`, attach the mirror promise instead of awaiting. */ + waitUntil?: (promise: Promise) => void }) { // Capture ownership and blob keys before D1 delete. A failed read must // abort the delete (and be retried) rather than orphan the R2 blobs; only @@ -1097,6 +1114,21 @@ export async function deleteEmailMessageById(input: { .prepare(`DELETE FROM email_messages WHERE id = ?`) .bind(input.messageId), ]) + // D1 is authoritative: mirror only after the batch commits, never before. + // Captured owner is required; system owners are skipped inside the helper. + if (row && input.env) { + const mirrorPromise = mirrorMailboxDeleteMessageMetadata({ + env: input.env, + ownerId: row.user_id, + messageId: input.messageId, + deletedAt: nowIso(), + }) + if (input.waitUntil) { + input.waitUntil(mirrorPromise) + } else { + await mirrorPromise + } + } for (const blobKey of blobKeys) { await input.blobs.delete(blobKey).catch((error: unknown) => { console.warn('email-blob-delete-failed', blobKey, error) diff --git a/packages/worker/src/email/service-attachment-cleanup.node.test.ts b/packages/worker/src/email/service-attachment-cleanup.node.test.ts new file mode 100644 index 0000000000..6a7d072c14 --- /dev/null +++ b/packages/worker/src/email/service-attachment-cleanup.node.test.ts @@ -0,0 +1,139 @@ +import { expect, test, vi } from 'vitest' +import { silenceExpectedConsoleErrors } from '#worker/test-support/console-spies.ts' +import type * as EmailRepo from './repo.ts' + +const mocks = vi.hoisted(() => ({ + insertEmailMessage: vi.fn(), + insertEmailAttachments: vi.fn(), + deleteEmailMessageById: vi.fn(async () => undefined), + getEmailMessageById: vi.fn(), + withAccountWriteLease: vi.fn( + async (input: { write: () => Promise }) => input.write(), + ), +})) + +vi.mock('#worker/account/deletion-state.ts', () => ({ + withAccountWriteLease: mocks.withAccountWriteLease, +})) + +vi.mock('./repo.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + insertEmailMessage: mocks.insertEmailMessage, + insertEmailAttachments: mocks.insertEmailAttachments, + deleteEmailMessageById: mocks.deleteEmailMessageById, + getEmailMessageById: mocks.getEmailMessageById, + } +}) + +const { insertEmailMessageWithAttachments, RetryableInboundStorageError } = + await import('./service.ts') + +test('attachment cleanup forwards optional Mailbox env and waitUntil to delete', async () => { + const db = {} as D1Database + const blobs = { + put: vi.fn(async () => ({})), + delete: vi.fn(async () => undefined), + } as unknown as R2Bucket + const env = { APP_DB: db } as Env + const waitUntil = vi.fn() + const storedMessage = { + id: 'message-1', + userId: 'user-1', + direction: 'inbound' as const, + rawMimeKey: 'email/user-1/message-1/raw.eml', + } + + mocks.insertEmailMessage.mockResolvedValueOnce(storedMessage) + mocks.insertEmailAttachments.mockRejectedValueOnce( + new Error('simulated attachment insert failure'), + ) + mocks.getEmailMessageById.mockResolvedValueOnce(null) + + await expect( + insertEmailMessageWithAttachments({ + db, + blobs, + env, + waitUntil, + message: { + id: 'message-1', + direction: 'inbound', + userId: 'user-1', + rawMime: 'cleanup-bytes', + processingStatus: 'stored', + }, + attachments: [ + { + filename: 'note.txt', + contentType: 'text/plain', + storageKind: 'raw-mime', + storageKey: null, + }, + ], + }), + ).rejects.toBeInstanceOf(RetryableInboundStorageError) + + expect(mocks.deleteEmailMessageById).toHaveBeenCalledWith({ + db, + blobs, + messageId: 'message-1', + env, + waitUntil, + }) +}) + +test('attachment cleanup without mirror context keeps acknowledge/retry rules', async () => { + silenceExpectedConsoleErrors(['inbound-email-attachment-cleanup-failed']) + const db = {} as D1Database + const blobs = { + put: vi.fn(async () => ({})), + delete: vi.fn(async () => undefined), + } as unknown as R2Bucket + const remaining = { + id: 'message-2', + userId: 'user-2', + direction: 'inbound' as const, + rawMimeKey: 'email/user-2/message-2/raw.eml', + } + + mocks.insertEmailMessage.mockResolvedValueOnce(remaining) + mocks.insertEmailAttachments.mockRejectedValueOnce( + new Error('simulated attachment insert failure'), + ) + mocks.deleteEmailMessageById.mockRejectedValueOnce( + new Error('simulated cleanup failure'), + ) + mocks.getEmailMessageById.mockResolvedValueOnce(remaining) + + await expect( + insertEmailMessageWithAttachments({ + db, + blobs, + message: { + id: 'message-2', + direction: 'inbound', + userId: 'user-2', + rawMime: 'orphan-bytes', + processingStatus: 'stored', + }, + attachments: [ + { + filename: 'note.txt', + contentType: 'text/plain', + storageKind: 'raw-mime', + storageKey: null, + }, + ], + }), + ).resolves.toMatchObject({ id: 'message-2' }) + + expect(mocks.deleteEmailMessageById).toHaveBeenCalledWith({ + db, + blobs, + messageId: 'message-2', + env: undefined, + waitUntil: undefined, + }) +}) diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index 2dc1d41770..6e81bd85cc 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -13,6 +13,7 @@ import { mirrorMailboxMessageGraphFromD1, type MailboxLiveMirrorEnv, } from './mailbox-live-mirror.ts' +import { type MailboxMirrorEnv } from './mailbox-mirror.ts' import { recordEmailReportingEvent, type EmailReportingEnv, @@ -216,6 +217,13 @@ export async function getEmailMessageWithAttachmentsById(input: { export async function insertEmailMessageWithAttachments( input: InsertEmailMessageWithRawMimeInput & { attachments: Parameters[0]['attachments'] + /** + * Optional Mailbox mirror context forwarded to cleanup deletes so an + * inbound agent can pass `env` / `waitUntil` without changing + * acknowledge/retry rules on ambiguous attachment failures. + */ + env?: MailboxMirrorEnv + waitUntil?: (promise: Promise) => void }, ) { let message @@ -243,6 +251,8 @@ export async function insertEmailMessageWithAttachments( db: input.db, blobs: input.blobs, messageId: message.id, + env: input.env, + waitUntil: input.waitUntil, }) } catch (error) { cleanupError = error From 6181071fd430781b23246115b6d5d62d1862a0e8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 13:35:59 +0000 Subject: [PATCH 22/62] Remove dormant delete dual-write from repo/service Restore deleteEmailMessageById to D1 batch then immediate R2 cleanup with no Mailbox env/waitUntil/mirror. Restore insertEmailMessageWithAttachments signature without mirror forwarding. Drop PR-only delete mirror tests and update data-storage.md: live explicit/retention deletes are repaired by parity purge/rebuild; direct delete wiring remains pending. Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 91 ++++--- .../repo-delete-mailbox-mirror.node.test.ts | 236 ------------------ packages/worker/src/email/repo.ts | 32 --- .../service-attachment-cleanup.node.test.ts | 139 ----------- packages/worker/src/email/service.ts | 10 - 5 files changed, 42 insertions(+), 466 deletions(-) delete mode 100644 packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts delete mode 100644 packages/worker/src/email/service-attachment-cleanup.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 609466919e..97e18a29a5 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -220,10 +220,9 @@ Durable Object export behavior: RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires live dual-write for outbound terminals, provider delivery-queue graph repair, - user classification (`service.ts#setEmailMessageClassification`), high-risk - inbound terminal paths, and `deleteEmailMessageById` metadata delete without - changing D1 authority — D1 `email_*` rows remain the live source of truth and - are still exported in the `d1` section. See + user classification (`service.ts#setEmailMessageClassification`), and high-risk + inbound terminal paths without changing D1 authority — D1 `email_*` rows remain + the live source of truth and are still exported in the `d1` section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. @@ -518,15 +517,13 @@ rows only. - Message deletes always delete the deterministic `emailRawMimeKey(userId, messageId)` from R2 (production writers always store that canonical key). `deleteEmailMessageById` runs an atomic D1 batch - (attachments, then message), then best-effort - `mirrorMailboxDeleteMessageMetadata` when `env` is supplied (metadata-only; - nulls delivery-event `message_id`, never deletes R2 or empty threads — thread - cleanup stays deferred), then best-effort R2 blob deletes in the same order as - before expand. Mirror failures/timeouts never affect return/throw; optional - `waitUntil` attaches the mirror without blocking R2. User email retention and - system-email retention stay strict: blob delete before row delete; failed blob - deletes skip the row for retry. Account deletion stays strict before atomic D1 - finalization; a failed blob delete preserves every message row for retry. + (attachments, then message), then best-effort R2 blob deletes. Live explicit + and retention deletes do not call Mailbox mirror helpers today; the parity lane + repairs DO state via purge/rebuild. Direct delete wiring is pending. User email + retention and system-email retention stay strict: blob delete before row delete; + failed blob deletes skip the row for retry. Account deletion stays strict before + atomic D1 finalization; a failed blob delete preserves every message row for + retry. - Bucket names: `kody-email-blobs` (production), per-preview `{worker}-email-blobs` buckets created and cleaned up by `tools/ci/preview-resources.ts`, and the test env reuses the preview-style @@ -712,10 +709,9 @@ Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment **Partial mutation RPCs** (owner-bound, monotonic on `updatedAt`, no R2 or retention-alarm side effects — unlike full snapshot mirrors, these do not mark -retention dirty or reschedule alarms). `deleteMessageMetadata` is live on the -`deleteEmailMessageById` path when `env` is supplied; touch/update/classify and -the other delete RPCs remain library-only on live paths that prefer full graph -repair: +retention dirty or reschedule alarms). All partial mutation RPCs remain +library-only on live paths that prefer full graph repair or parity backfill; +direct delete wiring for explicit/retention deletes is pending: - `touchThread` — advance `last_message_at` / `updated_at` without a full snapshot; `last_message_at` never moves backward @@ -776,10 +772,9 @@ never propagate into D1 commit paths. Helpers cover full snapshots (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). Partial -mutation helpers are library-only except `mirrorMailboxDeleteMessageMetadata` on -the `deleteEmailMessageById` path; other live paths prefer the graph -orchestrator below. D1 remains sole authority for all live mail read/write -paths. +mutation helpers are library-only; live paths prefer the graph orchestrator +below or parity purge/rebuild for deletes. D1 remains sole authority for all +live mail read/write paths. **Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 message graph (optional caller thread, message, attachments, then delivery @@ -949,24 +944,23 @@ classification (full graph repair after D1 update via `service.ts#setEmailMessageClassification`), high-risk inbound terminal paths (received graph + rejected delivery-event mirror + post-effects event re-mirror; `waitUntil`; no Mailbox before D1/R2 finalization; pre-claim bounded rejections -parity-lane only; `system:email` excluded), and `deleteEmailMessageById` -metadata delete (mirror only after the D1 batch; R2 unchanged; thread cleanup -deferred). Mirror helpers emit automatic namespaced outcome telemetry (`missing` -and `timeout` included). Message and batch event repair each use one 1s-bounded -RPC; each graph attempt emits at most two AE writes; event repair batches up to -100 events with explicit truncation (newest retained, chronological insertion -restored, stable truncation warning) and one batch telemetry outcome. The -every-5-minute queue-isolated `mailbox_parity` lane backfills all owner messages -and delivery events (including pre-claim bounded rejection rows), durable -content-watermark replays, compares owner-scoped D1 vs Mailbox counts, persists -soak state on `users` (migration `0125`), and re-purges the DO when account -deletion races the lane. Retention sweeper metadata-delete mirrors and live read -cutover are **still not wired**. Live email read/write authority is unchanged — +parity-lane only; `system:email` excluded). Mirror helpers emit automatic +namespaced outcome telemetry (`missing` and `timeout` included). Message and +batch event repair each use one 1s-bounded RPC; each graph attempt emits at most +two AE writes; event repair batches up to 100 events with explicit truncation +(newest retained, chronological insertion restored, stable truncation warning) and +one batch telemetry outcome. Live explicit and retention deletes do not call +Mailbox mirror helpers; the every-5-minute queue-isolated `mailbox_parity` lane +backfills all owner messages and delivery events (including pre-claim bounded +rejection rows), durable content-watermark replays, compares owner-scoped D1 vs +Mailbox counts, persists soak state on `users` (migration `0125`), re-purges the +DO when account deletion races the lane, and repairs delete drift via purge/rebuild. +Direct delete wiring is pending. Live email read/write authority is unchanged — D1 `email_*` tables remain authoritative for all live paths. D1 email rows and -existing R2 inventory deletion stay authoritative during expand. **Phase 2 -contract completion** (every user-mail D1 mutation also writes the DO on live -paths, including retention deletes) remains pending; terminal inbound + parity -cover the high-risk live surface today. +existing R2 inventory deletion stay authoritative during expand. **Phase 2 contract +completion** (every user-mail D1 mutation also writes the DO on live paths, +including retention deletes) remains pending; terminal inbound + parity cover the +high-risk live surface today. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -989,15 +983,15 @@ cover the high-risk live surface today. paths in `inbound.ts` (received graph via `waitUntil` only after D1/R2 + finalization; rejected delivery-event mirror; post-effects event re-mirror; already-received idempotent repair; pre-claim bounded rejections parity-lane - only; `system:email` excluded), and `deleteEmailMessageById` metadata delete - (mirror after D1 batch only) call the live mirror helpers; graph repair uses + only; `system:email` excluded) call the live mirror helpers; graph repair uses message RPC + one batch event RPC (at most two AE writes per attempt). The every-5-minute `mailbox_parity` scheduled lane backfills all owner messages - and delivery events, durable content-watermark replays, count compares, and - soak tracking on `users` (migration `0125`). **Still pending for phase-2 - contract completion:** retention sweeper metadata-delete mirrors. D1 - authority and read cutover are unchanged (prepared adapter exists; not - wired). + and delivery events, durable content-watermark replays, count compares, soak + tracking on `users` (migration `0125`), and repairs delete drift via + purge/rebuild. **Still pending for phase-2 contract completion:** direct delete + wiring for explicit/retention deletes (including retention sweeper + metadata-delete mirrors). D1 authority and read cutover are unchanged + (prepared adapter exists; not wired). 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified (`mailbox_parity_matching_since` ≥ 24h continuous exact counts, fresh @@ -1060,8 +1054,7 @@ failures acknowledge rather than risk duplicates. - **Ambiguity:** if attachment commit fails but message cleanup (or a residual probe) cannot prove the pre-commit state, acknowledge the already-created message (logged, non-retry) rather than risking a duplicate on Email Routing - retry. `deleteEmailMessageById` mirrors metadata only after the D1 batch; R2 - delete behavior is unchanged; empty-thread cleanup stays deferred. + retry. Empty-thread cleanup stays deferred. When Mailbox becomes read-authoritative (phase 3+), the same shape applies with the DO as the metadata store: @@ -1191,8 +1184,8 @@ Bindings are configured per environment in `packages/worker/wrangler.jsonc` reconciliation alarms) - `MAILBOX` (Durable Objects; per-user email metadata — see [Mailbox](#durable-objects-mailbox); phase 1 registers purge/export - consumption; phase 2 wires terminal inbound + outbound live dual-write and - `deleteEmailMessageById` metadata delete without changing D1 authority) + consumption; phase 2 wires terminal inbound + outbound live dual-write without + changing D1 authority) - `STORAGE_RUNNER` (Durable Objects) - `REPO_SESSION` (Durable Objects) - `PACKAGE_REALTIME_SESSION` (Durable Objects) diff --git a/packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts b/packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts deleted file mode 100644 index 804c2ed120..0000000000 --- a/packages/worker/src/email/repo-delete-mailbox-mirror.node.test.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { expect, test, vi } from 'vitest' -import { emailRawMimeKey } from './blob-keys.ts' - -const mocks = vi.hoisted(() => ({ - mirrorMailboxDeleteMessageMetadata: vi.fn(async () => ({ - status: 'mirrored' as const, - })), -})) - -vi.mock('./mailbox-mirror.ts', () => ({ - mirrorMailboxDeleteMessageMetadata: mocks.mirrorMailboxDeleteMessageMetadata, -})) - -const { deleteEmailMessageById } = await import('./repo.ts') - -function createDeleteHarness(input?: { - userId?: string | null - attachmentKeys?: Array - batchError?: Error -}) { - const userId = input?.userId === undefined ? 'user-1' : input.userId - const attachmentKeys = input?.attachmentKeys ?? ['attachments/user-1/a1'] - const callOrder: Array = [] - const deletedBlobKeys: Array = [] - - const selectUser = { - bind: vi.fn(() => ({ - first: vi.fn(async () => (userId == null ? null : { user_id: userId })), - })), - } - const selectAttachments = { - bind: vi.fn(() => ({ - all: vi.fn(async () => ({ - results: attachmentKeys.map((storage_key) => ({ storage_key })), - })), - })), - } - const deleteAttachments = { - bind: vi.fn(() => ({ kind: 'delete-attachments' })), - } - const deleteMessage = { - bind: vi.fn(() => ({ kind: 'delete-message' })), - } - - let prepareCalls = 0 - const db = { - prepare: vi.fn((sql: string) => { - prepareCalls += 1 - if (sql.includes('SELECT user_id')) return selectUser - if (sql.includes('SELECT storage_key')) return selectAttachments - if (sql.includes('DELETE FROM email_attachments')) - return deleteAttachments - if (sql.includes('DELETE FROM email_messages')) return deleteMessage - throw new Error(`unexpected prepare: ${sql}`) - }), - batch: vi.fn(async () => { - callOrder.push('d1-batch') - if (input?.batchError) throw input.batchError - return [] - }), - } as unknown as D1Database - - const blobs = { - delete: vi.fn(async (key: string) => { - callOrder.push(`r2:${key}`) - deletedBlobKeys.push(key) - }), - } as unknown as R2Bucket - - const env = { APP_DB: db } as unknown as Env - - return { - db, - blobs, - env, - callOrder, - deletedBlobKeys, - prepareCalls: () => prepareCalls, - messageId: 'message-1', - userId, - attachmentKeys, - } -} - -test('deleteEmailMessageById keeps D1-then-R2 order and mirrors only after D1', async () => { - const harness = createDeleteHarness() - mocks.mirrorMailboxDeleteMessageMetadata.mockImplementation(async () => { - harness.callOrder.push('mirror') - return { status: 'mirrored' } - }) - - await deleteEmailMessageById({ - db: harness.db, - blobs: harness.blobs, - messageId: harness.messageId, - env: harness.env, - }) - - expect(harness.callOrder[0]).toBe('d1-batch') - expect(harness.callOrder[1]).toBe('mirror') - expect(harness.callOrder.slice(2)).toEqual([ - `r2:${emailRawMimeKey('user-1', harness.messageId)}`, - 'r2:attachments/user-1/a1', - ]) - expect(mocks.mirrorMailboxDeleteMessageMetadata).toHaveBeenCalledTimes(1) - expect(mocks.mirrorMailboxDeleteMessageMetadata).toHaveBeenCalledWith({ - env: harness.env, - ownerId: 'user-1', - messageId: harness.messageId, - deletedAt: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T/), - }) - expect(harness.deletedBlobKeys).toEqual([ - emailRawMimeKey('user-1', harness.messageId), - 'attachments/user-1/a1', - ]) -}) - -test('deleteEmailMessageById without mirror env preserves D1/R2 behavior and skips DO delete', async () => { - const harness = createDeleteHarness() - - await deleteEmailMessageById({ - db: harness.db, - blobs: harness.blobs, - messageId: harness.messageId, - }) - - expect(harness.callOrder).toEqual([ - 'd1-batch', - `r2:${emailRawMimeKey('user-1', harness.messageId)}`, - 'r2:attachments/user-1/a1', - ]) - expect(mocks.mirrorMailboxDeleteMessageMetadata).not.toHaveBeenCalled() -}) - -test('deleteEmailMessageById skips mirror when the message row is absent', async () => { - const harness = createDeleteHarness({ userId: null, attachmentKeys: [] }) - - await deleteEmailMessageById({ - db: harness.db, - blobs: harness.blobs, - messageId: harness.messageId, - env: harness.env, - }) - - expect(harness.callOrder).toEqual(['d1-batch']) - expect(mocks.mirrorMailboxDeleteMessageMetadata).not.toHaveBeenCalled() - expect(harness.deletedBlobKeys).toEqual([]) -}) - -test('deleteEmailMessageById never throws when mirror reports failure or timeout', async () => { - const harness = createDeleteHarness({ attachmentKeys: [] }) - mocks.mirrorMailboxDeleteMessageMetadata - .mockResolvedValueOnce({ - status: 'error', - error: new Error('mailbox delete mirror failed'), - }) - .mockResolvedValueOnce({ status: 'timeout' }) - - await expect( - deleteEmailMessageById({ - db: harness.db, - blobs: harness.blobs, - messageId: harness.messageId, - env: harness.env, - }), - ).resolves.toBeUndefined() - - await expect( - deleteEmailMessageById({ - db: harness.db, - blobs: harness.blobs, - messageId: harness.messageId, - env: harness.env, - }), - ).resolves.toBeUndefined() - - expect(harness.deletedBlobKeys).toEqual([ - emailRawMimeKey('user-1', harness.messageId), - emailRawMimeKey('user-1', harness.messageId), - ]) -}) - -test('deleteEmailMessageById schedules mirror via waitUntil without awaiting before R2', async () => { - const harness = createDeleteHarness({ attachmentKeys: [] }) - let resolveMirror!: (value: { status: 'mirrored' }) => void - const mirrorPromise = new Promise<{ status: 'mirrored' }>((resolve) => { - resolveMirror = resolve - }) - mocks.mirrorMailboxDeleteMessageMetadata.mockReturnValueOnce(mirrorPromise) - - const waitUntilPromises: Array> = [] - const waitUntil = vi.fn((promise: Promise) => { - waitUntilPromises.push(promise) - }) - - const pending = deleteEmailMessageById({ - db: harness.db, - blobs: harness.blobs, - messageId: harness.messageId, - env: harness.env, - waitUntil, - }) - - await expect(pending).resolves.toBeUndefined() - expect(waitUntil).toHaveBeenCalledTimes(1) - expect(harness.callOrder).toEqual([ - 'd1-batch', - `r2:${emailRawMimeKey('user-1', harness.messageId)}`, - ]) - expect(harness.deletedBlobKeys).toEqual([ - emailRawMimeKey('user-1', harness.messageId), - ]) - - resolveMirror({ status: 'mirrored' }) - await Promise.all(waitUntilPromises) - expect(mocks.mirrorMailboxDeleteMessageMetadata).toHaveBeenCalledTimes(1) -}) - -test('deleteEmailMessageById does not mirror when D1 batch fails', async () => { - const harness = createDeleteHarness({ - batchError: new Error('simulated d1 batch failure'), - }) - - await expect( - deleteEmailMessageById({ - db: harness.db, - blobs: harness.blobs, - messageId: harness.messageId, - env: harness.env, - }), - ).rejects.toThrow('simulated d1 batch failure') - - expect(mocks.mirrorMailboxDeleteMessageMetadata).not.toHaveBeenCalled() - expect(harness.deletedBlobKeys).toEqual([]) - expect(harness.callOrder).toEqual(['d1-batch']) -}) diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 4072490f9a..0327befafc 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -4,10 +4,6 @@ import { utf8ByteLength, } from '@kody-internal/shared/backup-restore-safety.ts' import { parseJsonArray } from '@kody-internal/shared/json-parsing.ts' -import { - mirrorMailboxDeleteMessageMetadata, - type MailboxMirrorEnv, -} from './mailbox-mirror.ts' import { emailClassificationValues, type EmailAttachmentRecord, @@ -1064,15 +1060,6 @@ export async function searchEmailMessages(input: { return (result.results ?? []).map(mapMessageRow) } -/** - * Delete a message and its attachments from D1, then best-effort delete R2 - * blobs. When `env` is provided and the row existed, schedule a Mailbox - * metadata-delete mirror only after the authoritative D1 batch succeeds - * (`deletedAt` = canonical now). Mirror never affects return/throw; when - * `waitUntil` is supplied the mirror promise is attached there, otherwise it - * is awaited (bounded, never-throw). Absent rows skip DO delete. Thread - * cleanup stays deferred (RPC nulls delivery `message_id` only). - */ export async function deleteEmailMessageById(input: { db: D1Database /** @@ -1081,10 +1068,6 @@ export async function deleteEmailMessageById(input: { */ blobs: R2Bucket messageId: string - /** Optional Mailbox binding for post-D1 best-effort metadata delete. */ - env?: MailboxMirrorEnv - /** When set with `env`, attach the mirror promise instead of awaiting. */ - waitUntil?: (promise: Promise) => void }) { // Capture ownership and blob keys before D1 delete. A failed read must // abort the delete (and be retried) rather than orphan the R2 blobs; only @@ -1114,21 +1097,6 @@ export async function deleteEmailMessageById(input: { .prepare(`DELETE FROM email_messages WHERE id = ?`) .bind(input.messageId), ]) - // D1 is authoritative: mirror only after the batch commits, never before. - // Captured owner is required; system owners are skipped inside the helper. - if (row && input.env) { - const mirrorPromise = mirrorMailboxDeleteMessageMetadata({ - env: input.env, - ownerId: row.user_id, - messageId: input.messageId, - deletedAt: nowIso(), - }) - if (input.waitUntil) { - input.waitUntil(mirrorPromise) - } else { - await mirrorPromise - } - } for (const blobKey of blobKeys) { await input.blobs.delete(blobKey).catch((error: unknown) => { console.warn('email-blob-delete-failed', blobKey, error) diff --git a/packages/worker/src/email/service-attachment-cleanup.node.test.ts b/packages/worker/src/email/service-attachment-cleanup.node.test.ts deleted file mode 100644 index 6a7d072c14..0000000000 --- a/packages/worker/src/email/service-attachment-cleanup.node.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { expect, test, vi } from 'vitest' -import { silenceExpectedConsoleErrors } from '#worker/test-support/console-spies.ts' -import type * as EmailRepo from './repo.ts' - -const mocks = vi.hoisted(() => ({ - insertEmailMessage: vi.fn(), - insertEmailAttachments: vi.fn(), - deleteEmailMessageById: vi.fn(async () => undefined), - getEmailMessageById: vi.fn(), - withAccountWriteLease: vi.fn( - async (input: { write: () => Promise }) => input.write(), - ), -})) - -vi.mock('#worker/account/deletion-state.ts', () => ({ - withAccountWriteLease: mocks.withAccountWriteLease, -})) - -vi.mock('./repo.ts', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - insertEmailMessage: mocks.insertEmailMessage, - insertEmailAttachments: mocks.insertEmailAttachments, - deleteEmailMessageById: mocks.deleteEmailMessageById, - getEmailMessageById: mocks.getEmailMessageById, - } -}) - -const { insertEmailMessageWithAttachments, RetryableInboundStorageError } = - await import('./service.ts') - -test('attachment cleanup forwards optional Mailbox env and waitUntil to delete', async () => { - const db = {} as D1Database - const blobs = { - put: vi.fn(async () => ({})), - delete: vi.fn(async () => undefined), - } as unknown as R2Bucket - const env = { APP_DB: db } as Env - const waitUntil = vi.fn() - const storedMessage = { - id: 'message-1', - userId: 'user-1', - direction: 'inbound' as const, - rawMimeKey: 'email/user-1/message-1/raw.eml', - } - - mocks.insertEmailMessage.mockResolvedValueOnce(storedMessage) - mocks.insertEmailAttachments.mockRejectedValueOnce( - new Error('simulated attachment insert failure'), - ) - mocks.getEmailMessageById.mockResolvedValueOnce(null) - - await expect( - insertEmailMessageWithAttachments({ - db, - blobs, - env, - waitUntil, - message: { - id: 'message-1', - direction: 'inbound', - userId: 'user-1', - rawMime: 'cleanup-bytes', - processingStatus: 'stored', - }, - attachments: [ - { - filename: 'note.txt', - contentType: 'text/plain', - storageKind: 'raw-mime', - storageKey: null, - }, - ], - }), - ).rejects.toBeInstanceOf(RetryableInboundStorageError) - - expect(mocks.deleteEmailMessageById).toHaveBeenCalledWith({ - db, - blobs, - messageId: 'message-1', - env, - waitUntil, - }) -}) - -test('attachment cleanup without mirror context keeps acknowledge/retry rules', async () => { - silenceExpectedConsoleErrors(['inbound-email-attachment-cleanup-failed']) - const db = {} as D1Database - const blobs = { - put: vi.fn(async () => ({})), - delete: vi.fn(async () => undefined), - } as unknown as R2Bucket - const remaining = { - id: 'message-2', - userId: 'user-2', - direction: 'inbound' as const, - rawMimeKey: 'email/user-2/message-2/raw.eml', - } - - mocks.insertEmailMessage.mockResolvedValueOnce(remaining) - mocks.insertEmailAttachments.mockRejectedValueOnce( - new Error('simulated attachment insert failure'), - ) - mocks.deleteEmailMessageById.mockRejectedValueOnce( - new Error('simulated cleanup failure'), - ) - mocks.getEmailMessageById.mockResolvedValueOnce(remaining) - - await expect( - insertEmailMessageWithAttachments({ - db, - blobs, - message: { - id: 'message-2', - direction: 'inbound', - userId: 'user-2', - rawMime: 'orphan-bytes', - processingStatus: 'stored', - }, - attachments: [ - { - filename: 'note.txt', - contentType: 'text/plain', - storageKind: 'raw-mime', - storageKey: null, - }, - ], - }), - ).resolves.toMatchObject({ id: 'message-2' }) - - expect(mocks.deleteEmailMessageById).toHaveBeenCalledWith({ - db, - blobs, - messageId: 'message-2', - env: undefined, - waitUntil: undefined, - }) -}) diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index 6e81bd85cc..2dc1d41770 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -13,7 +13,6 @@ import { mirrorMailboxMessageGraphFromD1, type MailboxLiveMirrorEnv, } from './mailbox-live-mirror.ts' -import { type MailboxMirrorEnv } from './mailbox-mirror.ts' import { recordEmailReportingEvent, type EmailReportingEnv, @@ -217,13 +216,6 @@ export async function getEmailMessageWithAttachmentsById(input: { export async function insertEmailMessageWithAttachments( input: InsertEmailMessageWithRawMimeInput & { attachments: Parameters[0]['attachments'] - /** - * Optional Mailbox mirror context forwarded to cleanup deletes so an - * inbound agent can pass `env` / `waitUntil` without changing - * acknowledge/retry rules on ambiguous attachment failures. - */ - env?: MailboxMirrorEnv - waitUntil?: (promise: Promise) => void }, ) { let message @@ -251,8 +243,6 @@ export async function insertEmailMessageWithAttachments( db: input.db, blobs: input.blobs, messageId: message.id, - env: input.env, - waitUntil: input.waitUntil, }) } catch (error) { cleanupError = error From d195a6807baeab13986dace4d38cf8ae3c87c428 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 13:39:48 +0000 Subject: [PATCH 23/62] fix(email): serialize inbound Mailbox terminal work Co-authored-by: Kent C. Dodds --- .../inbound-mailbox-mirror.workers.test.ts | 224 ++++++++++++++++-- .../src/email/inbound-mailbox.node.test.ts | 212 +++++++++++++++++ packages/worker/src/email/inbound-mailbox.ts | 118 +++++++++ packages/worker/src/email/inbound.ts | 109 ++------- 4 files changed, 547 insertions(+), 116 deletions(-) create mode 100644 packages/worker/src/email/inbound-mailbox.node.test.ts create mode 100644 packages/worker/src/email/inbound-mailbox.ts diff --git a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts index 6cfb4c3ec1..9bb2f63c29 100644 --- a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts +++ b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts @@ -21,11 +21,27 @@ function createInboundEnv() { return { ...env, APP_BASE_URL: platformBaseUrl } } -async function seedAccount(input: { +function createCapturedWaitUntilContext() { + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + return { ctx, waitUntilPromises } +} + +async function drainWaitUntil(waitUntilPromises: Array>) { + for (let index = 0; index < waitUntilPromises.length; index += 1) { + await waitUntilPromises[index] + } +} + +async function seedVerifiedAccount(input: { db: D1Database email: string username: string - emailVerifiedAt?: string | null }) { const stableUserId = await createStableUserIdFromEmail(input.email) await input.db @@ -43,23 +59,13 @@ async function seedAccount(input: { input.username, input.email, 'test-password-hash', - input.emailVerifiedAt === undefined - ? new Date().toISOString() - : input.emailVerifiedAt, + new Date().toISOString(), stableUserId, 'max', ) .run() } -async function seedVerifiedAccount(input: { - db: D1Database - email: string - username: string -}) { - await seedAccount(input) -} - async function readUserDailyReceiveCount(userId: string) { const result = await userMeterRpc({ env, userId }).read({ resource: 'email_receives_per_day', @@ -160,6 +166,7 @@ test( }) const mailbox = rpcFor(userId) await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() const raw = buildInboundAttachmentRaw({ address, @@ -173,8 +180,9 @@ test( to: address, raw, }) - await handleInboundEmail(message, createInboundEnv()) + await handleInboundEmail(message, createInboundEnv(), ctx) expect(message.rejectedReason).toBeNull() + await drainWaitUntil(waitUntilPromises) const [stored] = await listEmailMessages({ db: env.APP_DB, @@ -219,11 +227,131 @@ test( id: expect.stringMatching(/^email-inbound-delivery:/), provider: 'cloudflare-email-routing', }) + expect(mirroredEvents[0]?.usageEffectRecordedAt).toEqual(expect.any(String)) expect(await readUserDailyReceiveCount(userId)).toBe(1) }, inboundMailboxMirrorTimeoutMs, ) +test( + 'delayed graph RPCs cannot overwrite post-effects delivery event fields', + async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + await ensureUsageRollupsTestSchema(env.APP_DB) + const username = `mbx-ord-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-ord-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + + const real = rpcFor(userId) + await real.getMessage({ messageId: 'warmup-nonexistent' }) + let releaseGraph!: () => void + const graphGate = new Promise((resolve) => { + releaseGraph = resolve + }) + const order: Array = [] + const delayedStub = { + async mirrorMessage(...args: Parameters) { + order.push('graph-message') + await graphGate + return await real.mirrorMessage(...args) + }, + async upsertDeliveryEvents( + ...args: Parameters + ) { + order.push('graph-batch') + await graphGate + return await real.upsertDeliveryEvents(...args) + }, + async upsertDeliveryEvent( + ...args: Parameters + ) { + order.push('post-effects-event') + return await real.upsertDeliveryEvent(...args) + }, + } + + const orderedEnv = { + ...createInboundEnv(), + MAILBOX: { + idFromName: (name: string) => env.MAILBOX.idFromName(name), + get: () => delayedStub, + } as unknown as DurableObjectNamespace, + } + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Ordered terminal mirror', + 'Message-ID: ', + '', + 'Order body', + ].join('\r\n') + const message = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + + await handleInboundEmail(message, orderedEnv, ctx) + expect(message.rejectedReason).toBeNull() + expect(waitUntilPromises.length).toBeGreaterThanOrEqual(1) + + // Wait until the waitUntil task reaches the gated graph RPC. + const graphStartedAt = Date.now() + while (!order.includes('graph-message')) { + if (Date.now() - graphStartedAt > 5_000) { + throw new Error('Timed out waiting for gated graph mirror RPC') + } + await new Promise((resolve) => setTimeout(resolve, 10)) + } + expect(order).toEqual(['graph-message']) + expect(order.includes('post-effects-event')).toBe(false) + const [stored] = await listEmailMessages({ + db: env.APP_DB, + userId, + limit: 1, + }) + expect(stored).toBeDefined() + if (!stored) throw new Error('Expected stored inbound message') + const preEffectsLedger = await env.APP_DB.prepare( + `SELECT detail_json FROM email_delivery_events + WHERE user_id = ? AND id LIKE 'email-inbound-delivery:%' + AND event_type = 'received'`, + ) + .bind(userId) + .first<{ detail_json: string }>() + expect( + JSON.parse(preEffectsLedger!.detail_json).usageEffectRecordedAt ?? null, + ).toBeNull() + + releaseGraph() + await drainWaitUntil(waitUntilPromises) + + expect(order).toEqual([ + 'graph-message', + 'graph-batch', + 'post-effects-event', + ]) + + const mailbox = rpcFor(userId) + const mirroredEvents = await mailbox.listDeliveryEvents({ + messageId: stored.id, + limit: 10, + }) + expect(mirroredEvents).toHaveLength(1) + expect(mirroredEvents[0]?.usageEffectRecordedAt).toEqual(expect.any(String)) + expect(mirroredEvents[0]?.subscriptionEffectState).toBe('complete') + }, + inboundMailboxMirrorTimeoutMs, +) + test( 'Email Routing retry creates one message and charge with idempotent Mailbox mirror', async () => { @@ -258,8 +386,12 @@ test( to: address, raw, }) - await handleInboundEmail(first, createInboundEnv()) - await handleInboundEmail(second, createInboundEnv()) + const firstCtx = createCapturedWaitUntilContext() + const secondCtx = createCapturedWaitUntilContext() + await handleInboundEmail(first, createInboundEnv(), firstCtx.ctx) + await drainWaitUntil(firstCtx.waitUntilPromises) + await handleInboundEmail(second, createInboundEnv(), secondCtx.ctx) + await drainWaitUntil(secondCtx.waitUntilPromises) expect(first.rejectedReason).toBeNull() expect(second.rejectedReason).toBeNull() @@ -316,16 +448,20 @@ test( ...createInboundEnv(), EMAIL_BLOBS: createFailingEmailBlobs(), } as Parameters[1] + const failCtx = createCapturedWaitUntilContext() const first = createForwardableEmailMessage({ from: 'sender@example.net', to: address, raw, }) - await expect(handleInboundEmail(first, failingEnv)).rejects.toBeInstanceOf( - RetryableInboundStorageError, - ) + await expect( + handleInboundEmail(first, failingEnv, failCtx.ctx), + ).rejects.toBeInstanceOf(RetryableInboundStorageError) expect(first.rejectedReason).toBeNull() + // Charge may schedule UserMeter D1 mirror via waitUntil; no Mailbox + // terminal work should have produced a message below. + await drainWaitUntil(failCtx.waitUntilPromises) expect(await readUserDailyReceiveCount(userId)).toBe(1) expect( await listEmailMessages({ db: env.APP_DB, userId, limit: 10 }), @@ -343,12 +479,14 @@ test( await mailbox.getMessage({ messageId: candidate.messageId }), ).toBeNull() + const retryCtx = createCapturedWaitUntilContext() const retry = createForwardableEmailMessage({ from: 'sender@example.net', to: address, raw, }) - await handleInboundEmail(retry, createInboundEnv()) + await handleInboundEmail(retry, createInboundEnv(), retryCtx.ctx) + await drainWaitUntil(retryCtx.waitUntilPromises) expect(retry.rejectedReason).toBeNull() expect(await readUserDailyReceiveCount(userId)).toBe(1) const [stored] = await listEmailMessages({ @@ -378,6 +516,7 @@ test( 'mailbox-live-mirror-delivery-event-failed', ]) await ensureEmailTestSchema(env.APP_DB) + await ensureUsageRollupsTestSchema(env.APP_DB) { const username = `mbx-err-${crypto.randomUUID().slice(0, 8)}` @@ -402,15 +541,27 @@ test( to: address, raw, }) + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() await handleInboundEmail( message, createInboundMailboxStubEnv({ mode: 'throw' }), + ctx, ) expect(message.rejectedReason).toBeNull() + await drainWaitUntil(waitUntilPromises) expect(await readUserDailyReceiveCount(userId)).toBe(1) expect( await listEmailMessages({ db: env.APP_DB, userId, limit: 1 }), ).toHaveLength(1) + const ledger = await env.APP_DB.prepare( + `SELECT json_extract(detail_json, '$.usageEffectRecordedAt') AS recorded + FROM email_delivery_events + WHERE user_id = ? AND event_type = 'received'`, + ) + .bind(userId) + .first<{ recorded: string | null }>() + // Graph failure/timeout must not skip D1 effects. + expect(ledger?.recorded).toEqual(expect.any(String)) } { @@ -436,19 +587,31 @@ test( to: address, raw, }) + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() const startedAt = Date.now() await handleInboundEmail( message, createInboundMailboxStubEnv({ mode: 'hang' }), + ctx, ) - const elapsedMs = Date.now() - startedAt + // Handler acknowledges before the bounded hang finishes. + expect(Date.now() - startedAt).toBeLessThan(mailboxMirrorRpcTimeoutMs) expect(message.rejectedReason).toBeNull() + await drainWaitUntil(waitUntilPromises) + const elapsedMs = Date.now() - startedAt expect(await readUserDailyReceiveCount(userId)).toBe(1) expect( await listEmailMessages({ db: env.APP_DB, userId, limit: 1 }), ).toHaveLength(1) - // Graph + post-effects event each race the mirror RPC bound. expect(elapsedMs).toBeLessThan(mailboxMirrorRpcTimeoutMs * 8) + const ledger = await env.APP_DB.prepare( + `SELECT json_extract(detail_json, '$.usageEffectRecordedAt') AS recorded + FROM email_delivery_events + WHERE user_id = ? AND event_type = 'received'`, + ) + .bind(userId) + .first<{ recorded: string | null }>() + expect(ledger?.recorded).toEqual(expect.any(String)) } }, inboundMailboxMirrorTimeoutMs, @@ -465,6 +628,7 @@ test( 'mailbox-live-mirror-delivery-event-failed', ]) await ensureEmailTestSchema(env.APP_DB) + await ensureUsageRollupsTestSchema(env.APP_DB) const username = `mbx-replay-${crypto.randomUUID().slice(0, 8)}` const accountEmail = `mbx-replay-${crypto.randomUUID()}@example.com` const userId = await createStableUserIdFromEmail(accountEmail) @@ -484,6 +648,7 @@ test( body: 'Stored before repair.', attachmentText: 'Repair me', }) + const firstCtx = createCapturedWaitUntilContext() const first = createForwardableEmailMessage({ from: 'sender@example.net', to: address, @@ -492,7 +657,9 @@ test( await handleInboundEmail( first, createInboundMailboxStubEnv({ mode: 'throw' }), + firstCtx.ctx, ) + await drainWaitUntil(firstCtx.waitUntilPromises) expect(first.rejectedReason).toBeNull() const [stored] = await listEmailMessages({ db: env.APP_DB, @@ -504,12 +671,14 @@ test( expect(await mailbox.getMessage({ messageId: stored.id })).toBeNull() expect(await readUserDailyReceiveCount(userId)).toBe(1) + const replayCtx = createCapturedWaitUntilContext() const replay = createForwardableEmailMessage({ from: 'sender@example.net', to: address, raw, }) - await handleInboundEmail(replay, createInboundEnv()) + await handleInboundEmail(replay, createInboundEnv(), replayCtx.ctx) + await drainWaitUntil(replayCtx.waitUntilPromises) expect(replay.rejectedReason).toBeNull() expect(await readUserDailyReceiveCount(userId)).toBe(1) expect( @@ -568,12 +737,14 @@ test( .spyOn(parser, 'parseForwardableEmailRawMime') .mockRejectedValueOnce(new Error('simulated parse failure')) try { + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() const message = createForwardableEmailMessage({ from: 'sender@example.net', to: address, raw, }) - await handleInboundEmail(message, createInboundEnv()) + await handleInboundEmail(message, createInboundEnv(), ctx) + await drainWaitUntil(waitUntilPromises) expect(message.rejectedReason).toBe('simulated parse failure') expect(await readUserDailyReceiveCount(userId)).toBe(1) expect( @@ -602,13 +773,14 @@ test( messageId: null, }) - // Claimed-rejected replay re-mirrors the same event idempotently. + const replayCtx = createCapturedWaitUntilContext() const replay = createForwardableEmailMessage({ from: 'sender@example.net', to: address, raw, }) - await handleInboundEmail(replay, createInboundEnv()) + await handleInboundEmail(replay, createInboundEnv(), replayCtx.ctx) + await drainWaitUntil(replayCtx.waitUntilPromises) expect(replay.rejectedReason).toBe('simulated parse failure') const afterReplay = await mailbox.listDeliveryEvents({ messageId: null, diff --git a/packages/worker/src/email/inbound-mailbox.node.test.ts b/packages/worker/src/email/inbound-mailbox.node.test.ts new file mode 100644 index 0000000000..925a1c524e --- /dev/null +++ b/packages/worker/src/email/inbound-mailbox.node.test.ts @@ -0,0 +1,212 @@ +import { expect, test, vi } from 'vitest' +import { consoleError } from '#worker/test-support/console-spies.ts' +import { systemEmailOwnerId } from './email-owner.ts' + +const mocks = vi.hoisted(() => ({ + mirrorMailboxMessageGraphFromD1: vi.fn(async () => ({ + messageId: 'msg-1', + message: { status: 'mirrored' as const }, + events: [], + eventsTruncated: false, + })), + mirrorMailboxDeliveryEventFromD1: vi.fn(async () => ({ + status: 'mirrored' as const, + })), + processInboundDeliveryEffects: vi.fn(async () => ({ + outcome: 'complete' as const, + })), +})) + +vi.mock('./mailbox-live-mirror.ts', () => ({ + mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, + mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, +})) + +vi.mock('./inbound-effects.ts', () => ({ + processInboundDeliveryEffects: mocks.processInboundDeliveryEffects, +})) + +const { + scheduleInboundReceivedTerminalWork, + scheduleInboundRejectedTerminalWork, +} = await import('./inbound-mailbox.ts') + +function createCapturedWaitUntilContext() { + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + return { ctx, waitUntilPromises } +} + +function resetMocks() { + mocks.mirrorMailboxMessageGraphFromD1.mockReset() + mocks.mirrorMailboxDeliveryEventFromD1.mockReset() + mocks.processInboundDeliveryEffects.mockReset() + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue({ + messageId: 'msg-1', + message: { status: 'mirrored' as const }, + events: [], + eventsTruncated: false, + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ + status: 'mirrored' as const, + }) + mocks.processInboundDeliveryEffects.mockResolvedValue({ + outcome: 'complete' as const, + }) +} + +test('received terminal work orders graph then effects then event mirror via waitUntil', async () => { + resetMocks() + consoleError.mockImplementation(() => {}) + const order: Array = [] + let releaseGraph!: () => void + const graphGate = new Promise((resolve) => { + releaseGraph = resolve + }) + + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation(async () => { + order.push('graph-start') + await graphGate + order.push('graph-end') + return { + messageId: 'msg-1', + message: { status: 'mirrored' as const }, + events: [], + eventsTruncated: false, + } + }) + mocks.processInboundDeliveryEffects.mockImplementation(async () => { + order.push('effects') + return { outcome: 'complete' as const } + }) + mocks.mirrorMailboxDeliveryEventFromD1.mockImplementation(async () => { + order.push('event') + return { status: 'mirrored' as const } + }) + + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() + const env = { APP_DB: {} } as unknown as Parameters< + typeof scheduleInboundReceivedTerminalWork + >[0]['env'] + + await scheduleInboundReceivedTerminalWork({ + env, + userId: 'user-aaa', + messageId: 'msg-1', + deliveryId: 'delivery-1', + expectedFinalizationToken: 'token-1', + durationMs: 12, + ctx, + logLabel: 'Inbound email effect dispatch failed', + }) + + expect(waitUntilPromises).toHaveLength(1) + expect(order).toEqual(['graph-start']) + expect(mocks.processInboundDeliveryEffects).not.toHaveBeenCalled() + + releaseGraph() + await Promise.all(waitUntilPromises) + + expect(order).toEqual(['graph-start', 'graph-end', 'effects', 'event']) + expect(mocks.processInboundDeliveryEffects).toHaveBeenCalledWith({ + env, + userId: 'user-aaa', + deliveryId: 'delivery-1', + expectedFinalizationToken: 'token-1', + durationMs: 12, + waitUntil: expect.any(Function), + }) + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'user-aaa', + eventId: 'delivery-1', + }) +}) + +test('graph failure does not skip D1 effects; effects failure skips event mirror and logs once', async () => { + resetMocks() + consoleError.mockImplementation(() => {}) + + mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue({ + messageId: 'msg-1', + message: { status: 'timeout' as const }, + events: [], + eventsTruncated: false, + }) + mocks.processInboundDeliveryEffects.mockRejectedValueOnce( + new Error('effects exploded'), + ) + + await scheduleInboundReceivedTerminalWork({ + env: { APP_DB: {} } as unknown as Parameters< + typeof scheduleInboundReceivedTerminalWork + >[0]['env'], + userId: 'user-bbb', + messageId: 'msg-1', + deliveryId: 'delivery-2', + logLabel: 'Inbound email effect dispatch failed', + }) + + expect(mocks.processInboundDeliveryEffects).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxDeliveryEventFromD1).not.toHaveBeenCalled() + expect(consoleError).toHaveBeenCalledWith( + 'Inbound email effect dispatch failed', + expect.objectContaining({ message: 'effects exploded' }), + ) +}) + +test('received and rejected coordinators skip system:email owners', async () => { + resetMocks() + const env = { APP_DB: {} } as unknown as Parameters< + typeof scheduleInboundReceivedTerminalWork + >[0]['env'] + + await scheduleInboundReceivedTerminalWork({ + env, + userId: systemEmailOwnerId, + messageId: 'msg-sys', + deliveryId: 'delivery-sys', + logLabel: 'System should not run', + }) + await scheduleInboundRejectedTerminalWork({ + env, + userId: systemEmailOwnerId, + deliveryId: 'delivery-sys', + }) + + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + expect(mocks.processInboundDeliveryEffects).not.toHaveBeenCalled() + expect(mocks.mirrorMailboxDeliveryEventFromD1).not.toHaveBeenCalled() +}) + +test('rejected terminal schedules delivery-event mirror only', async () => { + resetMocks() + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() + const env = { APP_DB: {} } as unknown as Parameters< + typeof scheduleInboundRejectedTerminalWork + >[0]['env'] + + await scheduleInboundRejectedTerminalWork({ + env, + userId: 'user-ccc', + deliveryId: 'delivery-3', + ctx, + }) + + expect(waitUntilPromises).toHaveLength(1) + await Promise.all(waitUntilPromises) + expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() + expect(mocks.processInboundDeliveryEffects).not.toHaveBeenCalled() + expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith({ + env, + db: env.APP_DB, + userId: 'user-ccc', + eventId: 'delivery-3', + }) +}) diff --git a/packages/worker/src/email/inbound-mailbox.ts b/packages/worker/src/email/inbound-mailbox.ts new file mode 100644 index 0000000000..5076d63c1e --- /dev/null +++ b/packages/worker/src/email/inbound-mailbox.ts @@ -0,0 +1,118 @@ +import { isSystemEmailOwner } from './email-owner.ts' +import { processInboundDeliveryEffects } from './inbound-effects.ts' +import { + mirrorMailboxDeliveryEventFromD1, + mirrorMailboxMessageGraphFromD1, + type MailboxLiveMirrorEnv, +} from './mailbox-live-mirror.ts' +import { type EmailReportingEnv } from './reporting-events.ts' + +/** + * Env surface for inbound Mailbox terminal coordination. + * D1 remains authoritative; Mailbox dual-write is best-effort after commit. + */ +export type InboundMailboxEnv = Pick< + Env, + | 'APP_DB' + | 'BUNDLE_ARTIFACTS_KV' + | 'APP_BASE_URL' + | 'USAGE_EVENTS' + | 'MAILBOX' + | 'EMAIL_EVENTS' +> & + EmailReportingEnv & + MailboxLiveMirrorEnv + +export type InboundReceivedTerminalWorkInput = { + env: InboundMailboxEnv + userId: string + messageId: string + deliveryId: string + expectedFinalizationToken?: string + durationMs?: number + ctx?: ExecutionContext + logLabel: string +} + +export type InboundRejectedTerminalWorkInput = { + env: InboundMailboxEnv + userId: string + deliveryId: string + ctx?: ExecutionContext +} + +/** + * One ordered received-terminal task: + * 1) bounded full message graph mirror (never throws) + * 2) D1 inbound delivery effects + * 3) on effects success only, mirror the updated delivery event + * + * Attach to `ctx.waitUntil` when present; otherwise await. Catch/log once so + * Mailbox/effects failures never throw into Email Routing. + * `system:email` is skipped (caller keeps the effects-only system path). + */ +export async function scheduleInboundReceivedTerminalWork( + input: InboundReceivedTerminalWorkInput, +) { + if (isSystemEmailOwner(input.userId)) return + + const nestedWaitUntil = input.ctx + ? (promise: Promise) => { + input.ctx!.waitUntil(promise) + } + : undefined + + const task = (async () => { + await mirrorMailboxMessageGraphFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + messageId: input.messageId, + }) + await processInboundDeliveryEffects({ + env: input.env, + userId: input.userId, + deliveryId: input.deliveryId, + expectedFinalizationToken: input.expectedFinalizationToken, + durationMs: input.durationMs, + waitUntil: nestedWaitUntil, + }) + await mirrorMailboxDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + eventId: input.deliveryId, + }) + })().catch((error: unknown) => { + console.error(input.logLabel, error) + }) + + if (input.ctx) { + input.ctx.waitUntil(task) + return + } + await task +} + +/** + * Rejected-terminal delivery-event mirror only (no message graph). + * `system:email` is skipped. Never throws into Email Routing. + */ +export async function scheduleInboundRejectedTerminalWork( + input: InboundRejectedTerminalWorkInput, +) { + if (isSystemEmailOwner(input.userId)) return + + const task = mirrorMailboxDeliveryEventFromD1({ + env: input.env, + db: input.env.APP_DB, + userId: input.userId, + eventId: input.deliveryId, + }) + + if (input.ctx) { + input.ctx.waitUntil(task) + return + } + await task +} diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index b00aae8b49..2e8855eb0a 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -47,15 +47,15 @@ import { getPlatformEmailDomain, getSystemEmailDomain, } from './platform-address.ts' +import { + scheduleInboundReceivedTerminalWork, + scheduleInboundRejectedTerminalWork, + type InboundMailboxEnv, +} from './inbound-mailbox.ts' import { recordEmailReportingEvent, type EmailReportingEnv, } from './reporting-events.ts' -import { - mirrorMailboxDeliveryEventFromD1, - mirrorMailboxMessageGraphFromD1, - type MailboxLiveMirrorEnv, -} from './mailbox-live-mirror.ts' import { deleteEmptyEmailThreads, getEmailMessageById } from './repo.ts' import { recordBoundedEmailRejectionEvent, @@ -71,59 +71,6 @@ import { type SystemEmailLocal, } from './system-email.ts' -type InboundMailboxMirrorEnv = Pick< - Env, - 'APP_DB' | 'MAILBOX' | 'EMAIL_EVENTS' -> & - MailboxLiveMirrorEnv - -/** - * Best-effort D1 → Mailbox graph repair after durable inbound commit. - * Uses `ctx.waitUntil` when present; otherwise awaits the never-throw mirror. - * Must not run before D1/R2 storage + `markInboundDeliveryReceived`. - */ -async function scheduleInboundMailboxMessageGraphMirror(input: { - env: InboundMailboxMirrorEnv - userId: string - messageId: string - ctx?: ExecutionContext -}) { - const promise = mirrorMailboxMessageGraphFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: input.userId, - messageId: input.messageId, - }) - if (input.ctx) { - input.ctx.waitUntil(promise) - return - } - await promise -} - -/** - * Best-effort D1 → Mailbox delivery-event mirror (rejected / post-effects). - * Uses `ctx.waitUntil` when present; otherwise awaits the never-throw mirror. - */ -async function scheduleInboundMailboxDeliveryEventMirror(input: { - env: InboundMailboxMirrorEnv - userId: string - deliveryId: string - ctx?: ExecutionContext -}) { - const promise = mirrorMailboxDeliveryEventFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: input.userId, - eventId: input.deliveryId, - }) - if (input.ctx) { - input.ctx.waitUntil(promise) - return - } - await promise -} - /** * Rejection audit writes are best-effort (the SMTP reject already happened), * but a failure must still be visible to operators — silently losing the @@ -215,9 +162,9 @@ async function cleanupInboundDurability(input: { } } +/** System-inbox effects only (no Mailbox dual-write). */ async function scheduleInboundDeliveryEffects(input: { - env: Parameters[0]['env'] & - InboundMailboxMirrorEnv + env: Parameters[0]['env'] userId: string deliveryId: string expectedFinalizationToken?: string @@ -228,8 +175,6 @@ async function scheduleInboundDeliveryEffects(input: { const waitUntil = input.ctx ? (promise: Promise) => input.ctx!.waitUntil(promise) : undefined - // On success, re-mirror the updated delivery event (usage/subscription - // effect fields). Failures keep the existing log + reconcile path only. const promise = processInboundDeliveryEffects({ env: input.env, userId: input.userId, @@ -237,14 +182,6 @@ async function scheduleInboundDeliveryEffects(input: { expectedFinalizationToken: input.expectedFinalizationToken, durationMs: input.durationMs, waitUntil, - }).then(async () => { - if (input.userId === systemEmailOwnerId) return - await mirrorMailboxDeliveryEventFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: input.userId, - eventId: input.deliveryId, - }) }) if (input.ctx) { input.ctx.waitUntil( @@ -274,7 +211,7 @@ export async function handleInboundEmail( | 'EMAIL_EVENTS' > & EmailReportingEnv & - MailboxLiveMirrorEnv, + InboundMailboxEnv, ctx?: ExecutionContext, ) { const recipient = normalizeEmailAddress(message.to) @@ -665,7 +602,7 @@ export async function handleInboundEmail( message.setReject( claimedDelivery.rejectionReason ?? 'Failed to parse inbound email.', ) - await scheduleInboundMailboxDeliveryEventMirror({ + await scheduleInboundRejectedTerminalWork({ env, userId, deliveryId: claimedDelivery.deliveryId, @@ -680,16 +617,12 @@ export async function handleInboundEmail( messageId: claimedDelivery.messageId, }) if (existing) { - await scheduleInboundMailboxMessageGraphMirror({ + await scheduleInboundReceivedTerminalWork({ env, userId, messageId: claimedDelivery.messageId, - ctx, - }) - await scheduleInboundDeliveryEffects({ - env, - userId, deliveryId: claimedDelivery.deliveryId, + expectedFinalizationToken: claimedDelivery.finalizationToken, durationMs: Date.now() - receiveStartedAtMs, ctx, logLabel: 'Inbound email effect reconciliation failed', @@ -713,7 +646,7 @@ export async function handleInboundEmail( }) if (!rejected) return await recordReceiveUsage({ outcome: 'error' }) - await scheduleInboundMailboxDeliveryEventMirror({ + await scheduleInboundRejectedTerminalWork({ env, userId, deliveryId: claimedDelivery.deliveryId, @@ -729,11 +662,15 @@ export async function handleInboundEmail( }) if (!storageClaim.claimed) { if (storageClaim.delivery?.state === 'received') { - await scheduleInboundMailboxMessageGraphMirror({ + await scheduleInboundReceivedTerminalWork({ env, userId, messageId: claimedDelivery.messageId, + deliveryId: storageClaim.delivery.deliveryId, + expectedFinalizationToken: storageClaim.delivery.finalizationToken, + durationMs: Date.now() - receiveStartedAtMs, ctx, + logLabel: 'Inbound email effect reconciliation failed', }) return } @@ -764,15 +701,10 @@ export async function handleInboundEmail( } // Mailbox dual-write only after durable D1/R2 commit + finalization win. if (!storedResult.wonFinalization) return - await scheduleInboundMailboxMessageGraphMirror({ + await scheduleInboundReceivedTerminalWork({ env, userId, messageId: storedResult.finalizedDelivery.messageId, - ctx, - }) - await scheduleInboundDeliveryEffects({ - env, - userId, deliveryId: storedResult.finalizedDelivery.deliveryId, expectedFinalizationToken: storedResult.finalizedDelivery.finalizationToken, @@ -793,10 +725,7 @@ async function handleSystemInboundEmail(input: { | 'BUNDLE_ARTIFACTS_KV' | 'APP_BASE_URL' | 'USAGE_EVENTS' - | 'MAILBOX' - | 'EMAIL_EVENTS' - > & - MailboxLiveMirrorEnv + > recipient: string localPart: SystemEmailLocal systemDomain: string From 8c7c8fa98debb7cf0c7d78cc4426eec4b149d1d5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 13:47:09 +0000 Subject: [PATCH 24/62] docs: format inbound Mailbox architecture Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 97e18a29a5..83a653c8b7 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -220,10 +220,10 @@ Durable Object export behavior: RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires live dual-write for outbound terminals, provider delivery-queue graph repair, - user classification (`service.ts#setEmailMessageClassification`), and high-risk - inbound terminal paths without changing D1 authority — D1 `email_*` rows remain - the live source of truth and are still exported in the `d1` section. See - [Mailbox](#durable-objects-mailbox). + user classification (`service.ts#setEmailMessageClassification`), and + high-risk inbound terminal paths without changing D1 authority — D1 `email_*` + rows remain the live source of truth and are still exported in the `d1` + section. See [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -518,12 +518,12 @@ rows only. `emailRawMimeKey(userId, messageId)` from R2 (production writers always store that canonical key). `deleteEmailMessageById` runs an atomic D1 batch (attachments, then message), then best-effort R2 blob deletes. Live explicit - and retention deletes do not call Mailbox mirror helpers today; the parity lane - repairs DO state via purge/rebuild. Direct delete wiring is pending. User email - retention and system-email retention stay strict: blob delete before row delete; - failed blob deletes skip the row for retry. Account deletion stays strict before - atomic D1 finalization; a failed blob delete preserves every message row for - retry. + and retention deletes do not call Mailbox mirror helpers today; the parity + lane repairs DO state via purge/rebuild. Direct delete wiring is pending. User + email retention and system-email retention stay strict: blob delete before row + delete; failed blob deletes skip the row for retry. Account deletion stays + strict before atomic D1 finalization; a failed blob delete preserves every + message row for retry. - Bucket names: `kody-email-blobs` (production), per-preview `{worker}-email-blobs` buckets created and cleaned up by `tools/ci/preview-resources.ts`, and the test env reuses the preview-style @@ -948,19 +948,19 @@ parity-lane only; `system:email` excluded). Mirror helpers emit automatic namespaced outcome telemetry (`missing` and `timeout` included). Message and batch event repair each use one 1s-bounded RPC; each graph attempt emits at most two AE writes; event repair batches up to 100 events with explicit truncation -(newest retained, chronological insertion restored, stable truncation warning) and -one batch telemetry outcome. Live explicit and retention deletes do not call +(newest retained, chronological insertion restored, stable truncation warning) +and one batch telemetry outcome. Live explicit and retention deletes do not call Mailbox mirror helpers; the every-5-minute queue-isolated `mailbox_parity` lane backfills all owner messages and delivery events (including pre-claim bounded rejection rows), durable content-watermark replays, compares owner-scoped D1 vs Mailbox counts, persists soak state on `users` (migration `0125`), re-purges the -DO when account deletion races the lane, and repairs delete drift via purge/rebuild. -Direct delete wiring is pending. Live email read/write authority is unchanged — -D1 `email_*` tables remain authoritative for all live paths. D1 email rows and -existing R2 inventory deletion stay authoritative during expand. **Phase 2 contract -completion** (every user-mail D1 mutation also writes the DO on live paths, -including retention deletes) remains pending; terminal inbound + parity cover the -high-risk live surface today. +DO when account deletion races the lane, and repairs delete drift via +purge/rebuild. Direct delete wiring is pending. Live email read/write authority +is unchanged — D1 `email_*` tables remain authoritative for all live paths. D1 +email rows and existing R2 inventory deletion stay authoritative during expand. +**Phase 2 contract completion** (every user-mail D1 mutation also writes the DO +on live paths, including retention deletes) remains pending; terminal inbound + +parity cover the high-risk live surface today. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -983,14 +983,14 @@ high-risk live surface today. paths in `inbound.ts` (received graph via `waitUntil` only after D1/R2 + finalization; rejected delivery-event mirror; post-effects event re-mirror; already-received idempotent repair; pre-claim bounded rejections parity-lane - only; `system:email` excluded) call the live mirror helpers; graph repair uses - message RPC + one batch event RPC (at most two AE writes per attempt). The - every-5-minute `mailbox_parity` scheduled lane backfills all owner messages - and delivery events, durable content-watermark replays, count compares, soak - tracking on `users` (migration `0125`), and repairs delete drift via - purge/rebuild. **Still pending for phase-2 contract completion:** direct delete - wiring for explicit/retention deletes (including retention sweeper - metadata-delete mirrors). D1 authority and read cutover are unchanged + only; `system:email` excluded) call the live mirror helpers; graph repair + uses message RPC + one batch event RPC (at most two AE writes per attempt). + The every-5-minute `mailbox_parity` scheduled lane backfills all owner + messages and delivery events, durable content-watermark replays, count + compares, soak tracking on `users` (migration `0125`), and repairs delete + drift via purge/rebuild. **Still pending for phase-2 contract completion:** + direct delete wiring for explicit/retention deletes (including retention + sweeper metadata-delete mirrors). D1 authority and read cutover are unchanged (prepared adapter exists; not wired). 3. **Reads cut over after production soak** — user-mail reads move to the DO only after production parity soak is verified From 8397c903a3e4b84935af00225cd38f5adb31621b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 13:54:33 +0000 Subject: [PATCH 25/62] docs: name inbound Mailbox coordinators Co-authored-by: Kent C. Dodds --- docs/contributing/architecture/data-storage.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 83a653c8b7..1db517eba2 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -810,20 +810,20 @@ Never throws; returns a bounded summary. **Live callers:** - **Inbound terminals** (`inbound.ts`) — high-risk user-mail dual-write with strict ordering: **no Mailbox RPC before** durable D1/R2 message + attachment storage and `received` finalization win. The winner schedules full graph - repair via `ctx.waitUntil` (`scheduleInboundMailboxMessageGraphMirror`). + repair via `ctx.waitUntil` (`scheduleInboundReceivedTerminalWork`). **Already-received** Email Routing retries (delivery ledger `state === 'received'` with an existing message row) idempotently repair the graph and re-run effect reconciliation without a second charge. **Rejected terminals** (post-claim parse failure or replay of a claimed `rejected` delivery) mirror the delivery event only via - `scheduleInboundMailboxDeliveryEventMirror` - (`mirrorMailboxDeliveryEventFromD1`). After successful - `processInboundDeliveryEffects`, the effects chain re-mirrors the updated - delivery event (usage/subscription fields). Mirror failures, timeouts, and - hangs never affect SMTP reject/refund/retry/charge semantics. **Pre-claim - bounded rejection rows** (`recordBoundedEmailRejectionEvent` for verification, - suspension, sender-policy, size, entitlement, and system-limit gates before - delivery claim/charge) stay **D1-only on the live path** — the every-5-minute + `scheduleInboundRejectedTerminalWork` (`mirrorMailboxDeliveryEventFromD1`). + After successful `processInboundDeliveryEffects`, the same received + coordinator task re-mirrors the updated delivery event (usage/subscription + fields). Mirror failures, timeouts, and hangs never affect SMTP + reject/refund/retry/charge semantics. **Pre-claim bounded rejection rows** + (`recordBoundedEmailRejectionEvent` for verification, suspension, + sender-policy, size, entitlement, and system-limit gates before delivery + claim/charge) stay **D1-only on the live path** — the every-5-minute `mailbox_parity` lane backfills them. **`system:email` stays excluded** (no per-user Mailbox object). Retention sweeper deletes and other bulk metadata-delete mirrors are **still not wired** on live paths. Scheduled From 91cf45a8814fa9c10250d9c8d482d527101027b2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 15:03:47 +0000 Subject: [PATCH 26/62] fix(email): batch Mailbox parity event backfill Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 48 ++- .../src/email/mailbox-mirror.node.test.ts | 96 +++++ packages/worker/src/email/mailbox-mirror.ts | 20 +- .../email/mailbox-parity-phases.node.test.ts | 367 ++++++++++++++++++ .../worker/src/email/mailbox-parity-phases.ts | 98 +++-- ...ncile-cursor-backfill-content.node.test.ts | 53 +-- ...ncile-mismatch-deletion-error.node.test.ts | 44 ++- .../worker/src/email/mailbox-reconcile.ts | 13 +- .../email/mailbox-snapshot-repo.node.test.ts | 16 + .../worker/src/email/mailbox-snapshot-repo.ts | 53 +++ .../src/test-support/mailbox-reconcile.ts | 14 + 11 files changed, 720 insertions(+), 102 deletions(-) create mode 100644 packages/worker/src/email/mailbox-parity-phases.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 17ab92e7df..ab081153f0 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -753,7 +753,10 @@ field-by-field. **Best-effort mirror helpers** (`mailbox-mirror.ts`): non-throwing wrappers around the DO RPCs for D1-authoritative dual-write. Each awaited DO RPC is -bounded by `mailboxMirrorRpcTimeoutMs` (1 second). Each returns a structured +bounded by `mailboxMirrorRpcTimeoutMs` (1 second) by default. +`mirrorMailboxDeliveryEventSnapshots` accepts an optional `timeoutMs` override +(telemetry/outcomes unchanged) so the scheduled parity lane can use a longer +batch bound without slowing live dual-write. Each returns a structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, `{ status: 'stale' }`, `{ status: 'missing' }`, `{ status: 'timeout' }`, `{ status: 'skipped', reason }` (`system-email` | `mailbox-unconfigured` | @@ -846,7 +849,9 @@ mail (`email_messages` or `email_delivery_events`) are always eligible; discoverable even when D1 mail is empty so DO-only leftovers after the last-row delete can be purged and reconciled. Never-tracked empty users stay out. Per-user work shares a ~10s wall-clock budget (`mailboxParityTimeBudgetMs`) -across the batch. +across the batch. The five-minute cron cadence plus that budget is the +production convergence loop: each tick should make forward progress on the +oldest owners rather than soaking the lane on per-event RPCs. **Account-deletion races:** `loadUserParityState`, `persistUserParityProgress`, and `rotateCheckedAt` require `deleting_at IS NULL` (zero-row updates are @@ -865,10 +870,19 @@ Per user, the lane: graph via `mirrorMailboxMessageGraphFromD1` until complete or budget-exhausted. 3. **Bounded delivery-event backfill** — after messages complete, keyset-pages - **every** owner `email_delivery_events` row by `(created_at, id)` — not only - `message_id`-null orphans — and mirrors each via - `mirrorMailboxParityDeliveryEventFromD1`. This repairs events omitted when - per-message graph mirroring truncates the event batch. + **every** owner `email_delivery_events` row by `(created_at, id)` into ready + `MailboxDeliveryEventInput` snapshots + (`listMailboxDeliveryEventMirrorInputsForOwnerKeyset`; not only + `message_id`-null orphans) and mirrors each page with **one** + `upsertDeliveryEvents` batch via `mirrorMailboxDeliveryEventSnapshots` + (`mailboxParityEventPageSize` ≤ DO max, timeout + `mailboxParityEventMirrorTimeoutMs` ≈ 5s). Production evidence: per-event 1s + RPCs repeatedly timed out on a lagging owner and prevented soak under the 10s + lane budget; page batches restore convergence while live dual-write keeps the + 1s bound. Cursor advances through per-event mirrored/stale/missing results + (equal `created_at` progresses by id); uniform timeout/error/unconfigured + retains the cursor so the next tick reloads the same page. Rows deleted + before the snapshot load simply do not appear. 4. **Durable content watermark replay** — after both creation phases complete, opens a frozen window `(watermark, upper]` (`mailbox_parity_content_replay_upper_at` set once when the window opens; @@ -1319,17 +1333,17 @@ Reconcile runs through the registry in cleanup, system-email retention, general retention, job retention, hourly usage-rollup aggregation, and the every-5-minute `mailbox_parity` lane (queue-isolated; up to sixteen users per tick within a ~10s budget; D1 user -discovery including previously tracked empty-mail owners, bounded message + full -delivery-event backfill, durable content-watermark replay, metadata-only purge + -full rebuild on count mismatch, and count compare — see -[Mailbox](#durable-objects-mailbox)). Each production queue message preserves -`scheduled_lane_failed` / D1 lock-contention log and Sentry context. A handled -lane failure is acknowledged and retried by the next cron tick, matching the old -cron semantics. A failed enqueue is reported and runs through the inline -fallback after all sibling enqueue attempts finish; multiple failed enqueues -fall back sequentially to avoid D1 lock contention. Consumer transport failures -retain the configured retry/DLQ behavior. No failure can abort or mask a sibling -invocation. +discovery including previously tracked empty-mail owners, bounded message + +page-batched delivery-event backfill (~5s batch timeout under the ~10s lane +budget), durable content-watermark replay, metadata-only purge + full rebuild on +count mismatch, and count compare — see [Mailbox](#durable-objects-mailbox)). +Each production queue message preserves `scheduled_lane_failed` / D1 +lock-contention log and Sentry context. A handled lane failure is acknowledged +and retried by the next cron tick, matching the old cron semantics. A failed +enqueue is reported and runs through the inline fallback after all sibling +enqueue attempts finish; multiple failed enqueues fall back sequentially to +avoid D1 lock contention. Consumer transport failures retain the configured +retry/DLQ behavior. No failure can abort or mask a sibling invocation. Production note: diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 29a6bae1e6..23ae74b51b 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -605,3 +605,99 @@ test('mailbox mirror batch helper maps timeout and error to every event', async vi.useRealTimers() } }) + +test('mailbox mirror batch helper honors optional timeoutMs override', async () => { + vi.useFakeTimers() + consoleWarn.mockImplementation(() => {}) + try { + const events = [baseDeliveryEventInput({ id: 'evt-slow' })] + let resolveRpc: + | ((value: { + results: Array<{ + eventId: string + inserted: boolean + accepted: boolean + }> + }) => void) + | undefined + const upsertDeliveryEvents = vi.fn( + () => + new Promise<{ + results: Array<{ + eventId: string + inserted: boolean + accepted: boolean + }> + }>((resolve) => { + resolveRpc = resolve + }), + ) + const { env, writeDataPoint } = fakeMailboxEnv({ upsertDeliveryEvents }) + + const defaultPending = mirrorMailboxDeliveryEventSnapshots({ + env, + ownerId: 'user-aaa', + events, + }) + const defaultExpectation = expect(defaultPending).resolves.toEqual([ + { eventId: 'evt-slow', result: { status: 'timeout' } }, + ]) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + await defaultExpectation + expect(writeDataPoint).toHaveBeenCalledTimes(1) + resolveRpc?.({ + results: [{ eventId: 'evt-slow', inserted: true, accepted: true }], + }) + + const overrideMs = 5_000 + let resolveOverride: + | ((value: { + results: Array<{ + eventId: string + inserted: boolean + accepted: boolean + }> + }) => void) + | undefined + const upsertOverride = vi.fn( + () => + new Promise<{ + results: Array<{ + eventId: string + inserted: boolean + accepted: boolean + }> + }>((resolve) => { + resolveOverride = resolve + }), + ) + const { env: overrideEnv, writeDataPoint: overrideWrite } = fakeMailboxEnv({ + upsertDeliveryEvents: upsertOverride, + }) + const overridePending = mirrorMailboxDeliveryEventSnapshots({ + env: overrideEnv, + ownerId: 'user-aaa', + events, + timeoutMs: overrideMs, + }) + await vi.advanceTimersByTimeAsync(mailboxMirrorRpcTimeoutMs) + resolveOverride?.({ + results: [{ eventId: 'evt-slow', inserted: true, accepted: true }], + }) + await expect(overridePending).resolves.toEqual([ + { eventId: 'evt-slow', result: { status: 'mirrored' } }, + ]) + expect(overrideWrite).toHaveBeenCalledTimes(1) + expect(overrideWrite).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'mirrored', + expect.any(String), + ], + }), + ) + } finally { + vi.useRealTimers() + } +}) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index b3869dd206..a5dbd44afa 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -41,10 +41,12 @@ import { * structured outcomes (`mirrored | stale | missing | timeout | skipped | * error`). `system:email` stays in D1 only. * - * Every awaited DO RPC is bounded by {@link mailboxMirrorRpcTimeoutMs}. Partial - * mutation `missing` stays distinct from `mirrored`; delete `missing` maps to - * `mirrored` because desired absence is achieved. Outcomes are recorded via - * `recordMailboxParityEvent` when a user id is known. + * Every awaited DO RPC is bounded by {@link mailboxMirrorRpcTimeoutMs} unless a + * helper accepts an explicit `timeoutMs` override (batch delivery-event mirror + * for scheduled parity). Partial mutation `missing` stays distinct from + * `mirrored`; delete `missing` maps to `mirrored` because desired absence is + * achieved. Outcomes are recorded via `recordMailboxParityEvent` when a user + * id is known. * * Delivery-event snapshots should be loaded via * `getMailboxDeliveryEventMirrorInput` (mailbox-snapshot-repo) so callers do @@ -322,14 +324,17 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { /** * Best-effort batch delivery-event snapshot mirror. * - * One 1s timeout and one `upsert_delivery_event_batch` telemetry outcome. - * Timeout/error/skip apply uniformly to each per-event summary entry. - * Empty `events` returns `[]` without RPC or telemetry. + * One timeout (default {@link mailboxMirrorRpcTimeoutMs}) and one + * `upsert_delivery_event_batch` telemetry outcome. Timeout/error/skip apply + * uniformly to each per-event summary entry. Empty `events` returns `[]` + * without RPC or telemetry. */ export async function mirrorMailboxDeliveryEventSnapshots(input: { env: MailboxMirrorEnv ownerId: string events: Array + /** Override RPC timeout; defaults to {@link mailboxMirrorRpcTimeoutMs}. */ + timeoutMs?: number }): Promise> { const operation = 'upsert_delivery_event_batch' as const if (input.events.length === 0) return [] @@ -354,6 +359,7 @@ export async function mirrorMailboxDeliveryEventSnapshots(input: { ownerId: owner.ownerId, events: input.events, }), + input.timeoutMs ?? mailboxMirrorRpcTimeoutMs, ) if (!raced.ok) { const result = { status: 'timeout' as const } diff --git a/packages/worker/src/email/mailbox-parity-phases.node.test.ts b/packages/worker/src/email/mailbox-parity-phases.node.test.ts new file mode 100644 index 0000000000..a1901ed2fd --- /dev/null +++ b/packages/worker/src/email/mailbox-parity-phases.node.test.ts @@ -0,0 +1,367 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { + backfillEventsForUser, + mailboxParityEventMirrorTimeoutMs, + mailboxParityEventPageSize, +} from './mailbox-parity-phases.ts' +import { type MailboxParityUserState } from './mailbox-parity-repo.ts' +import { mailboxMirrorRpcTimeoutMs } from './mailbox-mirror.ts' +import type * as MailboxMirrorModule from './mailbox-mirror.ts' +import { ensureEmailTestSchema } from './test-schema.ts' +import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' + +const mailboxMirrorMocks = vi.hoisted(() => ({ + mirrorMailboxDeliveryEventSnapshots: vi.fn(), +})) + +vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + mirrorMailboxDeliveryEventSnapshots: + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots, + } +}) + +async function createPhaseDb() { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureUsersTestSchema({ db }) + await ensureEmailTestSchema(db) + return db +} + +async function seedOwnerWithEvents(input: { + db: D1Database + userId: string + events: Array<{ id: string; createdAt: string }> +}) { + await input.db + .prepare( + `INSERT INTO users ( + username, email, password_hash, stable_user_id + ) VALUES (?, ?, 'hash', ?)`, + ) + .bind( + `u-${input.userId.slice(0, 8)}`, + `${input.userId}@example.test`, + input.userId, + ) + .run() + for (const event of input.events) { + await input.db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, event_type, provider, detail_json, created_at + ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + ) + .bind(event.id, input.userId, event.createdAt) + .run() + } +} + +function baseState(userId: string): MailboxParityUserState { + return { + userId, + matchingSince: null, + mismatchCount: 0, + contentWatermarkAt: '2026-08-01T10:00:00.000Z', + contentReplayUpperAt: null, + contentReplayCursor: null, + messageCursor: { createdAt: '2026-07-01T00:00:00.000Z', id: 'msg-1' }, + messagesCompletedAt: '2026-08-01T10:00:00.000Z', + eventCursor: null, + eventsCompletedAt: null, + } +} + +function phaseEnv(db: D1Database) { + return { + APP_DB: db, + MAILBOX: {} as DurableObjectNamespace, + EMAIL_EVENTS: { + writeDataPoint: vi.fn(), + } as unknown as AnalyticsEngineDataset, + } +} + +test('event backfill mirrors one page with one batch RPC under the scheduled timeout', async () => { + consoleWarn.mockImplementation(() => {}) + const db = await createPhaseDb() + const userId = 'a'.repeat(64) + const events = Array.from({ length: 3 }, (_, index) => ({ + id: `evt-${index}`, + createdAt: `2026-07-01T00:00:0${index}.000Z`, + })) + await seedOwnerWithEvents({ db, userId, events }) + + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots.mockImplementation( + async (input: { events: Array<{ id: string }>; timeoutMs?: number }) => + input.events.map((event) => ({ + eventId: event.id, + result: { status: 'mirrored' as const }, + })), + ) + + const result = await backfillEventsForUser({ + env: phaseEnv(db), + state: baseState(userId), + deadlineMs: Date.now() + 10_000, + isDeleting: async () => false, + }) + + expect(result).toMatchObject({ + status: 'complete', + backfilled: 3, + state: { eventCursor: { id: 'evt-2' } }, + }) + expect( + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots, + ).toHaveBeenCalledTimes(1) + expect( + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots, + ).toHaveBeenCalledWith( + expect.objectContaining({ + ownerId: userId, + timeoutMs: mailboxParityEventMirrorTimeoutMs, + events: [ + expect.objectContaining({ id: 'evt-0' }), + expect.objectContaining({ id: 'evt-1' }), + expect.objectContaining({ id: 'evt-2' }), + ], + }), + ) + expect(mailboxParityEventPageSize).toBeLessThanOrEqual( + mailboxUpsertDeliveryEventsMax, + ) + expect(mailboxParityEventMirrorTimeoutMs).toBeGreaterThan( + mailboxMirrorRpcTimeoutMs, + ) + expect(mailboxParityEventMirrorTimeoutMs).toBe(5_000) +}) + +test('event backfill keeps cursor on uniform timeout and advances through partial stale', async () => { + consoleWarn.mockImplementation(() => {}) + const db = await createPhaseDb() + const userId = 'b'.repeat(64) + await seedOwnerWithEvents({ + db, + userId, + events: [ + { id: 'evt-a', createdAt: '2026-07-01T00:00:00.000Z' }, + { id: 'evt-b', createdAt: '2026-07-01T00:00:01.000Z' }, + { id: 'evt-c', createdAt: '2026-07-01T00:00:02.000Z' }, + ], + }) + + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots.mockResolvedValueOnce([ + { eventId: 'evt-a', result: { status: 'timeout' } }, + { eventId: 'evt-b', result: { status: 'timeout' } }, + { eventId: 'evt-c', result: { status: 'timeout' } }, + ]) + + const timedOut = await backfillEventsForUser({ + env: phaseEnv(db), + state: baseState(userId), + deadlineMs: Date.now() + 10_000, + isDeleting: async () => false, + }) + expect(timedOut).toEqual({ + status: 'retryable_failure', + backfilled: 0, + blockedReason: 'event timeout', + state: expect.objectContaining({ + eventCursor: null, + eventsCompletedAt: null, + }), + }) + + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots.mockResolvedValueOnce([ + { eventId: 'evt-a', result: { status: 'stale' } }, + { eventId: 'evt-b', result: { status: 'mirrored' } }, + { eventId: 'evt-c', result: { status: 'missing' } }, + ]) + const advanced = await backfillEventsForUser({ + env: phaseEnv(db), + state: timedOut.state, + deadlineMs: Date.now() + 10_000, + isDeleting: async () => false, + }) + expect(advanced).toMatchObject({ + status: 'complete', + backfilled: 3, + state: { + eventCursor: { + createdAt: '2026-07-01T00:00:02.000Z', + id: 'evt-c', + }, + eventsCompletedAt: expect.any(String), + }, + }) +}) + +test('event backfill progresses equal createdAt timestamps by id keyset', async () => { + consoleWarn.mockImplementation(() => {}) + const db = await createPhaseDb() + const userId = 'c'.repeat(64) + const sameAt = '2026-07-01T00:00:00.000Z' + await seedOwnerWithEvents({ + db, + userId, + events: [ + { id: 'evt-a', createdAt: sameAt }, + { id: 'evt-b', createdAt: sameAt }, + { id: 'evt-c', createdAt: sameAt }, + { id: 'evt-d', createdAt: '2026-07-01T00:00:01.000Z' }, + ], + }) + + const seenPages: Array> = [] + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots.mockImplementation( + async (input: { events: Array<{ id: string }> }) => { + seenPages.push(input.events.map((event) => event.id)) + return input.events.map((event) => ({ + eventId: event.id, + result: { status: 'mirrored' as const }, + })) + }, + ) + + const firstPage = await backfillEventsForUser({ + env: phaseEnv(db), + state: { + ...baseState(userId), + eventCursor: null, + }, + deadlineMs: Date.now() + 10_000, + isDeleting: async () => false, + }) + expect(seenPages[0]).toEqual(['evt-a', 'evt-b', 'evt-c', 'evt-d']) + expect(firstPage).toMatchObject({ + status: 'complete', + backfilled: 4, + state: { + eventCursor: { createdAt: '2026-07-01T00:00:01.000Z', id: 'evt-d' }, + }, + }) + + seenPages.length = 0 + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots.mockImplementation( + async (input: { events: Array<{ id: string }> }) => { + seenPages.push(input.events.map((event) => event.id)) + // Fail after the first equal-timestamp event so the next tick resumes + // at evt-b (same createdAt, greater id). + return input.events.map((event, index) => ({ + eventId: event.id, + result: + index === 0 + ? { status: 'mirrored' as const } + : { status: 'timeout' as const }, + })) + }, + ) + const partial = await backfillEventsForUser({ + env: phaseEnv(db), + state: { + ...baseState(userId), + eventCursor: null, + eventsCompletedAt: null, + }, + deadlineMs: Date.now() + 10_000, + isDeleting: async () => false, + }) + expect(partial).toMatchObject({ + status: 'retryable_failure', + backfilled: 1, + blockedReason: 'event timeout', + state: { + eventCursor: { createdAt: sameAt, id: 'evt-a' }, + }, + }) + + seenPages.length = 0 + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots.mockImplementation( + async (input: { events: Array<{ id: string }> }) => { + seenPages.push(input.events.map((event) => event.id)) + return input.events.map((event) => ({ + eventId: event.id, + result: { status: 'mirrored' as const }, + })) + }, + ) + const resumed = await backfillEventsForUser({ + env: phaseEnv(db), + state: partial.state, + deadlineMs: Date.now() + 10_000, + isDeleting: async () => false, + }) + expect(seenPages[0]).toEqual(['evt-b', 'evt-c', 'evt-d']) + expect(resumed).toMatchObject({ + status: 'complete', + backfilled: 3, + state: { + eventCursor: { createdAt: '2026-07-01T00:00:01.000Z', id: 'evt-d' }, + }, + }) +}) + +test('event backfill uses ~5s batch timeout so a page that exceeds 1s can still succeed', async () => { + consoleWarn.mockImplementation(() => {}) + const db = await createPhaseDb() + const userId = 'd'.repeat(64) + await seedOwnerWithEvents({ + db, + userId, + events: [{ id: 'evt-slow', createdAt: '2026-07-01T00:00:00.000Z' }], + }) + + const slowRpcMs = mailboxMirrorRpcTimeoutMs + 500 + expect(slowRpcMs).toBeGreaterThan(mailboxMirrorRpcTimeoutMs) + expect(slowRpcMs).toBeLessThan(mailboxParityEventMirrorTimeoutMs) + + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots.mockImplementation( + async (input: { timeoutMs?: number; events: Array<{ id: string }> }) => { + const timeoutMs = input.timeoutMs ?? mailboxMirrorRpcTimeoutMs + return input.events.map((event) => ({ + eventId: event.id, + result: + slowRpcMs > timeoutMs + ? { status: 'timeout' as const } + : { status: 'mirrored' as const }, + })) + }, + ) + + const liveBoundProbe = + await mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots({ + env: phaseEnv(db), + ownerId: userId, + events: [{ id: 'evt-slow' }], + timeoutMs: mailboxMirrorRpcTimeoutMs, + }) + expect(liveBoundProbe).toEqual([ + { eventId: 'evt-slow', result: { status: 'timeout' } }, + ]) + + const result = await backfillEventsForUser({ + env: phaseEnv(db), + state: baseState(userId), + deadlineMs: Date.now() + 10_000, + isDeleting: async () => false, + }) + expect(result).toMatchObject({ + status: 'complete', + backfilled: 1, + }) + expect( + mailboxMirrorMocks.mirrorMailboxDeliveryEventSnapshots, + ).toHaveBeenCalledWith( + expect.objectContaining({ + timeoutMs: mailboxParityEventMirrorTimeoutMs, + }), + ) +}) diff --git a/packages/worker/src/email/mailbox-parity-phases.ts b/packages/worker/src/email/mailbox-parity-phases.ts index a7daa3b7dd..fc68c25383 100644 --- a/packages/worker/src/email/mailbox-parity-phases.ts +++ b/packages/worker/src/email/mailbox-parity-phases.ts @@ -4,19 +4,38 @@ import { type MailboxLiveMirrorEnv, type MailboxLiveMirrorGraphSummary, } from './mailbox-live-mirror.ts' -import { type MailboxMirrorResult } from './mailbox-mirror.ts' +import { + mirrorMailboxDeliveryEventSnapshots, + type MailboxMirrorResult, +} from './mailbox-mirror.ts' import { listContentReplayPage, - listEventBackfillPage, listMessageBackfillPage, type MailboxParityContentCursor, type MailboxParityUserState, } from './mailbox-parity-repo.ts' +import { listMailboxDeliveryEventMirrorInputsForOwnerKeyset } from './mailbox-snapshot-repo.ts' +import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' export const mailboxParityMessagePageSize = 10 +/** + * Events per backfill page / single `upsertDeliveryEvents` RPC. + * Must stay ≤ {@link mailboxUpsertDeliveryEventsMax}. + */ export const mailboxParityEventPageSize = 25 export const mailboxParityContentPageSize = 10 +/** + * Scheduled parity event-batch RPC timeout. + * + * Live dual-write keeps the 1s mirror bound so request paths stay snappy. + * Production soak evidence: per-event 1s RPCs repeatedly timed out on one + * owner and blocked convergence under the ~10s lane budget. One page-sized + * batch with a ~5s bound lets a tick make progress without exceeding the + * lane budget or the DO batch max. + */ +export const mailboxParityEventMirrorTimeoutMs = 5_000 + export type MailboxParityPhaseEnv = MailboxLiveMirrorEnv & { APP_DB: D1Database } @@ -103,8 +122,8 @@ export function messageGraphAllowsCursorAdvance( /** * Event-phase cursor: mirrored/stale advance; missing advances (D1 row may have - * been deleted concurrently after selection); skipped (incl. unconfigured), - * timeout, and error must not advance. + * been deleted concurrently after selection, or the batch omitted an id); + * skipped (incl. unconfigured), timeout, and error must not advance. */ export function eventMirrorAllowsCursorAdvance( result: MailboxMirrorResult, @@ -218,6 +237,14 @@ export async function backfillMessagesForUser(input: { return { status: 'budget_exhausted', state, backfilled } } +/** + * Keyset-page ready delivery-event snapshots and mirror each page with one + * `upsertDeliveryEvents` batch RPC ({@link mailboxParityEventMirrorTimeoutMs}). + * + * Cursor advances through per-event mirrored/stale/missing results in page + * order (including equal `createdAt` by id). Uniform timeout/error/skip keep + * the cursor so the next tick reloads the same page. + */ export async function backfillEventsForUser(input: { env: MailboxParityPhaseEnv state: MailboxParityUserState @@ -234,11 +261,18 @@ export async function backfillEventsForUser(input: { } while (Date.now() < input.deadlineMs) { - const page = await listEventBackfillPage({ + if (await input.isDeleting()) { + return { status: 'deletion_started', state, backfilled } + } + + const page = await listMailboxDeliveryEventMirrorInputsForOwnerKeyset({ db: input.env.APP_DB, - userId: state.userId, + ownerId: state.userId, cursor: state.eventCursor, - limit: mailboxParityEventPageSize, + limit: Math.min( + mailboxParityEventPageSize, + mailboxUpsertDeliveryEventsMax, + ), }) if (page.length === 0) { state = { @@ -248,39 +282,41 @@ export async function backfillEventsForUser(input: { return { status: 'complete', state, backfilled } } + if (Date.now() >= input.deadlineMs) { + return { status: 'budget_exhausted', state, backfilled } + } + if (await input.isDeleting()) { + return { status: 'deletion_started', state, backfilled } + } + + const results = await mirrorMailboxDeliveryEventSnapshots({ + env: input.env, + ownerId: state.userId, + events: page, + timeoutMs: mailboxParityEventMirrorTimeoutMs, + }) + + if (await input.isDeleting()) { + return { status: 'deletion_started', state, backfilled } + } + let cursor = state.eventCursor - for (const row of page) { - if (Date.now() >= input.deadlineMs) { - state = { ...state, eventCursor: cursor } - return { status: 'budget_exhausted', state, backfilled } - } - if (await input.isDeleting()) { - state = { ...state, eventCursor: cursor } - return { status: 'deletion_started', state, backfilled } - } - const result = await mirrorMailboxParityDeliveryEventFromD1({ - env: input.env, - db: input.env.APP_DB, - userId: state.userId, - eventId: row.id, - }) - if (await input.isDeleting()) { - state = { ...state, eventCursor: cursor } - return { status: 'deletion_started', state, backfilled } - } - if (!eventMirrorAllowsCursorAdvance(result)) { + for (let index = 0; index < results.length; index += 1) { + const item = results[index]! + const event = page[index]! + if (!eventMirrorAllowsCursorAdvance(item.result)) { state = { ...state, eventCursor: cursor } return { status: 'retryable_failure', state, backfilled, blockedReason: - result.status === 'skipped' - ? `event skipped:${result.reason}` - : `event ${result.status}`, + item.result.status === 'skipped' + ? `event skipped:${item.result.reason}` + : `event ${item.result.status}`, } } - cursor = { createdAt: row.created_at, id: row.id } + cursor = { createdAt: event.createdAt, id: event.id } backfilled += 1 } state = { ...state, eventCursor: cursor } diff --git a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts index f5188db190..0c862c13a7 100644 --- a/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-cursor-backfill-content.node.test.ts @@ -8,6 +8,7 @@ import { listEventBackfillPage, listUsersForMailboxParity, mockDoCounts, + mockMirroredEventSnapshots, parkUser, parityEnv, readParityRow, @@ -19,6 +20,7 @@ import { consoleWarn } from '#worker/test-support/console-spies.ts' import { systemEmailOwnerId } from './email-owner.ts' import { mailboxParityContentPageSize, + mailboxParityEventMirrorTimeoutMs, mailboxParityMessagePageSize, } from './mailbox-parity-phases.ts' @@ -141,9 +143,7 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou return successGraph(input.messageId) }, ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() mockDoCounts({ threads: 0, messages: 0, @@ -200,7 +200,7 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou async (input: { messageId: string }) => successGraph(input.messageId), ) mocks.mirrorMailboxMessageGraphFromD1.mockClear() - mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventSnapshots.mockClear() mocks.recordMailboxParityEvent.mockClear() await parkUser(db, ownerB) const mismatchNow = new Date('2026-08-01T11:00:00.000Z') @@ -217,11 +217,16 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou mismatched: 1, failed: 0, }) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'evt-bound-1' }), - ) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith( - expect.objectContaining({ userId: ownerA, eventId: 'evt-orphan-1' }), + expect(mocks.mirrorMailboxDeliveryEventSnapshots).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxDeliveryEventSnapshots).toHaveBeenCalledWith( + expect.objectContaining({ + ownerId: ownerA, + events: expect.arrayContaining([ + expect.objectContaining({ id: 'evt-bound-1' }), + expect.objectContaining({ id: 'evt-orphan-1' }), + ]), + timeoutMs: mailboxParityEventMirrorTimeoutMs, + }), ) const afterMismatch = await readParityRow(db, ownerA) @@ -247,7 +252,7 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou createdAt: '2026-07-03T00:00:00.000Z', }) mocks.mirrorMailboxMessageGraphFromD1.mockClear() - mocks.mirrorMailboxDeliveryEventFromD1.mockClear() + mocks.mirrorMailboxDeliveryEventSnapshots.mockClear() mockDoCounts({ threads: 1, messages: mailboxParityMessagePageSize + 2, @@ -345,9 +350,7 @@ test('mailbox parity content watermark repairs equal-count updates and soak', as mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() mockDoCounts({ threads: 0, messages: 2, @@ -438,9 +441,7 @@ test('mailbox parity replays message updated during initial backfill', async () return successGraph(input.messageId) }, ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() mockDoCounts({ threads: 0, messages: 2, @@ -496,9 +497,7 @@ test('mailbox parity content replay resumes durable upper and cursor across tick mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() mockDoCounts({ threads: 0, messages: messageCount, @@ -599,9 +598,7 @@ test('mailbox parity eventsTruncated still completes via full event phase', asyn mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue( successGraph('msg-1', true), ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() mockDoCounts({ threads: 0, messages: 1, @@ -622,5 +619,15 @@ test('mailbox parity eventsTruncated still completes via full event phase', asyn mismatched: 0, failed: 0, }) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledTimes(3) + expect(mocks.mirrorMailboxDeliveryEventSnapshots).toHaveBeenCalledTimes(1) + expect(mocks.mirrorMailboxDeliveryEventSnapshots).toHaveBeenCalledWith( + expect.objectContaining({ + ownerId: owner, + events: [ + expect.objectContaining({ id: 'evt-0' }), + expect.objectContaining({ id: 'evt-1' }), + expect.objectContaining({ id: 'evt-2' }), + ], + }), + ) }) diff --git a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts index 2f41201db7..c4aedb4007 100644 --- a/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile-mismatch-deletion-error.node.test.ts @@ -6,6 +6,7 @@ import { insertUser, listUsersForMailboxParity, mockDoCounts, + mockMirroredEventSnapshots, parityEnv, readParityRow, reconcileMailboxParity, @@ -48,9 +49,7 @@ test('mailbox parity missing and unconfigured do not advance creation cursors', events: [], eventsTruncated: false, }) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() mockDoCounts({ threads: 0, messages: 2, @@ -71,10 +70,16 @@ test('mailbox parity missing and unconfigured do not advance creation cursors', mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValueOnce({ - status: 'skipped', - reason: 'mailbox-unconfigured', - }) + mocks.mirrorMailboxDeliveryEventSnapshots.mockImplementationOnce( + async (input: { events: Array<{ id: string }> }) => + input.events.map((event) => ({ + eventId: event.id, + result: { + status: 'skipped' as const, + reason: 'mailbox-unconfigured' as const, + }, + })), + ) await expect( reconcileMailboxParity({ env, @@ -120,9 +125,16 @@ test('mailbox parity event missing advances cursor while unconfigured does not', mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), ) - mocks.mirrorMailboxDeliveryEventFromD1 - .mockResolvedValueOnce({ status: 'missing' }) - .mockResolvedValueOnce({ status: 'mirrored' }) + mocks.mirrorMailboxDeliveryEventSnapshots.mockImplementationOnce( + async (input: { events: Array<{ id: string }> }) => + input.events.map((event) => ({ + eventId: event.id, + result: + event.id === 'evt-gone' + ? { status: 'missing' as const } + : { status: 'mirrored' as const }, + })), + ) mockDoCounts({ threads: 0, messages: 1, @@ -180,9 +192,7 @@ test('mailbox parity deletion race purges Mailbox and skips parity writes', asyn .run() return successGraph('msg-1') }) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() await expect( reconcileMailboxParity({ @@ -230,9 +240,7 @@ test('tracked empty user discovers DO-only leftover, purges, then reaches zero p mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() const purge = vi.fn(async () => ({ ok: true as const })) mockDoCounts( { @@ -350,9 +358,7 @@ test('mismatch purge failure clears soak and does not start rebuild reset', asyn mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( async (input: { messageId: string }) => successGraph(input.messageId), ) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored', - }) + mockMirroredEventSnapshots() mockDoCounts({ threads: 0, messages: 1, diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index d2ee662310..2553e6dc8f 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -13,6 +13,7 @@ import { backfillEventsForUser, backfillMessagesForUser, mailboxParityContentPageSize, + mailboxParityEventMirrorTimeoutMs, mailboxParityEventPageSize, mailboxParityMessagePageSize, mirrorMailboxParityDeliveryEventFromD1, @@ -36,11 +37,12 @@ import { type MailboxCountResult } from './mailbox-types.ts' * Bounded five-minute Mailbox backfill + count-parity reconciler. * * Discovers a small oldest-first page of non-deleting D1 mail owners (no DO - * enumeration), mirrors message graphs then every owner delivery event through - * existing live-mirror helpers (repairing graph truncation), durably - * keyset-replays messages updated after the content watermark, then compares - * owner-scoped D1 counts with `countMailbox`. D1 remains mail authority; this - * lane never mutates email_* rows and does not flip read authority. + * enumeration), mirrors message graphs then every owner delivery event as + * page-sized `upsertDeliveryEvents` batches (repairing graph truncation; + * {@link mailboxParityEventMirrorTimeoutMs}), durably keyset-replays messages + * updated after the content watermark, then compares owner-scoped D1 counts + * with `countMailbox`. D1 remains mail authority; this lane never mutates + * email_* rows and does not flip read authority. */ export const mailboxParityUserBatchSize = 16 @@ -49,6 +51,7 @@ export const mailboxParityLastErrorMaxBytes = 512 export { mailboxParityContentPageSize, + mailboxParityEventMirrorTimeoutMs, mailboxParityEventPageSize, mailboxParityMessagePageSize, mirrorMailboxParityDeliveryEventFromD1, diff --git a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts index d79608b39f..5c14cfb87e 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.node.test.ts @@ -6,6 +6,7 @@ import { getMailboxDeliveryEventMirrorInput, listEmailDeliveryEventMirrorProjectionsForMessage, listMailboxDeliveryEventMirrorInputsForMessage, + listMailboxDeliveryEventMirrorInputsForOwnerKeyset, } from './mailbox-snapshot-repo.ts' import { ensureEmailTestSchema } from './test-schema.ts' @@ -181,4 +182,19 @@ test('mailbox-snapshot-repo loads complete delivery-event projection and ready M limit: 1, }) expect(newestOnly.map((event) => event.id)).toEqual(['evt-2']) + + const ownerPage = await listMailboxDeliveryEventMirrorInputsForOwnerKeyset({ + db, + ownerId: 'user-aaa', + cursor: null, + limit: 10, + }) + expect(ownerPage.map((event) => event.id)).toEqual(['evt-1', 'evt-2']) + const afterFirst = await listMailboxDeliveryEventMirrorInputsForOwnerKeyset({ + db, + ownerId: 'user-aaa', + cursor: { createdAt: '2026-07-02T10:00:00.000Z', id: 'evt-1' }, + limit: 10, + }) + expect(afterFirst.map((event) => event.id)).toEqual(['evt-2']) }) diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index 03d69a8fd8..cd7ea51392 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -173,3 +173,56 @@ export async function listMailboxDeliveryEventMirrorInputsForMessage(input: { }), ) } + +/** + * Owner-scoped delivery-event creation keyset of ready Mailbox snapshots. + * + * Orders by `(created_at ASC, id ASC)` so equal timestamps progress by id. + * Uses each event's `created_at` as `sourceMutationAt` (immutable inserts). + * Rows deleted before this query simply do not appear — callers advance only + * over returned snapshots (or reload the same cursor after uniform failure). + */ +export async function listMailboxDeliveryEventMirrorInputsForOwnerKeyset(input: { + db: D1Database + ownerId: string + cursor: { createdAt: string; id: string } | null + limit: number +}): Promise> { + const result = + input.cursor == null + ? await input.db + .prepare( + `${deliveryEventMirrorSelect} + WHERE user_id = ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.ownerId, input.limit) + .all() + : await input.db + .prepare( + `${deliveryEventMirrorSelect} + WHERE user_id = ? + AND ( + created_at > ? + OR (created_at = ? AND id > ?) + ) + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.ownerId, + input.cursor.createdAt, + input.cursor.createdAt, + input.cursor.id, + input.limit, + ) + .all() + return (result.results ?? []).map((row) => { + const projection = mapDeliveryEventMirrorRow(row) + return toMailboxDeliveryEventInput({ + projection, + sourceMutationAt: projection.createdAt, + }) + }) +} diff --git a/packages/worker/src/test-support/mailbox-reconcile.ts b/packages/worker/src/test-support/mailbox-reconcile.ts index 0388df44bd..4daf1c384e 100644 --- a/packages/worker/src/test-support/mailbox-reconcile.ts +++ b/packages/worker/src/test-support/mailbox-reconcile.ts @@ -11,6 +11,7 @@ import { ensureEmailTestSchema } from '#worker/email/test-schema.ts' const mocks = vi.hoisted(() => ({ mirrorMailboxMessageGraphFromD1: vi.fn(), mirrorMailboxDeliveryEventFromD1: vi.fn(), + mirrorMailboxDeliveryEventSnapshots: vi.fn(), awaitMailboxMirrorRpc: vi.fn(), recordMailboxParityEvent: vi.fn(), mailboxRpc: vi.fn(), @@ -30,6 +31,8 @@ vi.mock('#worker/email/mailbox-mirror.ts', async (importOriginal) => { return { ...actual, awaitMailboxMirrorRpc: mocks.awaitMailboxMirrorRpc, + mirrorMailboxDeliveryEventSnapshots: + mocks.mirrorMailboxDeliveryEventSnapshots, } }) @@ -200,6 +203,17 @@ export function successGraph(messageId: string, eventsTruncated = false) { } } +/** Default event-backfill batch mock: every snapshot mirrors. */ +export function mockMirroredEventSnapshots() { + mocks.mirrorMailboxDeliveryEventSnapshots.mockImplementation( + async (input: { events: Array<{ id: string }> }) => + input.events.map((event) => ({ + eventId: event.id, + result: { status: 'mirrored' as const }, + })), + ) +} + export function mockDoCounts( counts: { threads: number From 9c1b9eb97678487a1a9b6e8de849411e73db9927 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 15:37:38 +0000 Subject: [PATCH 27/62] fix(email): extend scheduled Mailbox count timeout Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 8 +- ...ilbox-reconcile-count-timeout.node.test.ts | 188 ++++++++++++++++++ .../worker/src/email/mailbox-reconcile.ts | 6 +- 3 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 packages/worker/src/email/mailbox-reconcile-count-timeout.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index ab081153f0..60bb48c89f 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -891,8 +891,12 @@ Per user, the lane: ticks. The watermark advances to `upper` only when the full window succeeds. 5. **Count compare + AE signals** — runs only after the event phase and a completed content window. Owner-scoped D1 counts vs `Mailbox.countMailbox()` - for threads, messages, attachments, and delivery events. Each comparison - emits one `mailbox_parity:` Analytics Engine row with + for threads, messages, attachments, and delivery events (timeout + `mailboxParityCountTimeoutMs` ≈ 5s). Production evidence: after event + backfill completed, tracked owners repeatedly hit `countMailbox timed out` + under the live 1s mirror bound and could not advance soak; the scheduled lane + uses a longer count-only bound while live dual-write mirrors stay at 1s. Each + comparison emits one `mailbox_parity:` Analytics Engine row with `double2 = d1Count - doCount`. Parity progress persists on `users` (migration `0125-mailbox-parity-state.sql`). diff --git a/packages/worker/src/email/mailbox-reconcile-count-timeout.node.test.ts b/packages/worker/src/email/mailbox-reconcile-count-timeout.node.test.ts new file mode 100644 index 0000000000..d32f08fbe4 --- /dev/null +++ b/packages/worker/src/email/mailbox-reconcile-count-timeout.node.test.ts @@ -0,0 +1,188 @@ +import { + createParityDb, + getMailboxReconcileTestMocks, + insertEvent, + insertMessage, + insertUser, + mockDoCounts, + mockMirroredEventSnapshots, + parityEnv, + readParityRow, + reconcileMailboxParity, + successGraph, +} from '#worker/test-support/mailbox-reconcile.ts' +import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { + awaitMailboxMirrorRpc, + mailboxMirrorRpcTimeoutMs, +} from './mailbox-mirror.ts' +import { mailboxParityCountTimeoutMs } from './mailbox-reconcile.ts' + +const mocks = getMailboxReconcileTestMocks() + +function isMailboxCountResult(value: unknown): value is { + threads: number + messages: number + attachments: number + deliveryEvents: number +} { + return ( + value != null && + typeof value === 'object' && + 'threads' in value && + 'messages' in value && + 'attachments' in value && + 'deliveryEvents' in value + ) +} + +async function seedReadyForCompare(db: D1Database, owner: string) { + await insertUser({ db, userId: owner, checkedAt: null }) + await insertMessage({ + db, + userId: owner, + id: 'msg-1', + createdAt: '2026-07-01T00:00:00.000Z', + }) + await insertEvent({ + db, + userId: owner, + id: 'evt-1', + messageId: 'msg-1', + createdAt: '2026-07-01T00:00:01.000Z', + }) + mocks.mirrorMailboxMessageGraphFromD1.mockImplementation( + async (input: { messageId: string }) => successGraph(input.messageId), + ) + mockMirroredEventSnapshots() +} + +test('count compare uses ~5s timeout so a count that exceeds 1s can still succeed', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'n'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await seedReadyForCompare(db, owner) + + const slowRpcMs = mailboxMirrorRpcTimeoutMs + 500 + expect(slowRpcMs).toBeGreaterThan(mailboxMirrorRpcTimeoutMs) + expect(slowRpcMs).toBeLessThan(mailboxParityCountTimeoutMs) + + const counts = { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + } + mocks.mailboxRpc.mockImplementation(() => ({ + countMailbox: vi.fn(async () => counts), + purge: vi.fn(async () => ({ ok: true as const })), + })) + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise, timeoutMs?: number) => { + const timeout = timeoutMs ?? mailboxMirrorRpcTimeoutMs + const value = await promise + if (isMailboxCountResult(value) && slowRpcMs > timeout) { + return { ok: false as const, timedOut: true as const } + } + return { ok: true as const, value } + }, + ) + + const liveBoundProbe = await awaitMailboxMirrorRpc( + Promise.resolve(counts), + mailboxMirrorRpcTimeoutMs, + ) + expect(liveBoundProbe).toEqual({ ok: false, timedOut: true }) + + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ + scanned: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(mocks.awaitMailboxMirrorRpc).toHaveBeenCalledWith( + expect.any(Promise), + mailboxParityCountTimeoutMs, + ) +}) + +test('count compare timeout clears soak and retries on the next tick', async () => { + consoleWarn.mockImplementation(() => {}) + const { db } = await createParityDb() + const owner = 'o'.repeat(64) + const now = new Date('2026-08-01T10:00:00.000Z') + await seedReadyForCompare(db, owner) + + const counts = { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + } + mockDoCounts(counts) + const env = parityEnv(db) + await expect( + reconcileMailboxParity({ env, now, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1 }) + const afterMatch = await readParityRow(db, owner) + expect(afterMatch?.matchingSince).toBe(now.toISOString()) + + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise, timeoutMs?: number) => { + const value = await promise + if (isMailboxCountResult(value)) { + const timeout = timeoutMs ?? mailboxMirrorRpcTimeoutMs + if (timeout >= mailboxParityCountTimeoutMs) { + return { ok: false as const, timedOut: true as const } + } + } + return { ok: true as const, value } + }, + ) + consoleWarn.mockClear() + const timeoutNow = new Date('2026-08-01T11:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: timeoutNow, batchSize: 1 }), + ).resolves.toEqual({ + scanned: 1, + backfilled: 0, + compared: 0, + matched: 0, + mismatched: 0, + failed: 1, + }) + expect(consoleWarn).toHaveBeenCalledWith( + 'mailbox-parity-reconcile-user-failed', + owner, + expect.any(Error), + ) + const afterTimeout = await readParityRow(db, owner) + expect(afterTimeout).toMatchObject({ + matchingSince: null, + lastError: 'Mailbox countMailbox timed out', + checkedAt: timeoutNow.toISOString(), + }) + + mocks.awaitMailboxMirrorRpc.mockImplementation( + async (promise: Promise) => ({ + ok: true as const, + value: await promise, + }), + ) + const retryNow = new Date('2026-08-01T12:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: retryNow, batchSize: 1 }), + ).resolves.toMatchObject({ matched: 1, compared: 1, failed: 0 }) + const afterRetry = await readParityRow(db, owner) + expect(afterRetry).toMatchObject({ + matchingSince: retryNow.toISOString(), + lastError: null, + mismatchCount: 0, + }) +}) diff --git a/packages/worker/src/email/mailbox-reconcile.ts b/packages/worker/src/email/mailbox-reconcile.ts index 2553e6dc8f..3e4cbdcc90 100644 --- a/packages/worker/src/email/mailbox-reconcile.ts +++ b/packages/worker/src/email/mailbox-reconcile.ts @@ -41,12 +41,14 @@ import { type MailboxCountResult } from './mailbox-types.ts' * page-sized `upsertDeliveryEvents` batches (repairing graph truncation; * {@link mailboxParityEventMirrorTimeoutMs}), durably keyset-replays messages * updated after the content watermark, then compares owner-scoped D1 counts - * with `countMailbox`. D1 remains mail authority; this lane never mutates + * with `countMailbox` ({@link mailboxParityCountTimeoutMs}). D1 remains mail + * authority; this lane never mutates * email_* rows and does not flip read authority. */ export const mailboxParityUserBatchSize = 16 export const mailboxParityTimeBudgetMs = 10_000 +export const mailboxParityCountTimeoutMs = 5_000 export const mailboxParityLastErrorMaxBytes = 512 export { @@ -252,7 +254,7 @@ async function compareParityForUser(input: { }) const raced = await awaitMailboxMirrorRpc( mailboxRpc({ env: input.env, userId: input.state.userId }).countMailbox(), - mailboxMirrorRpcTimeoutMs, + mailboxParityCountTimeoutMs, ) if (!raced.ok) { throw new Error('Mailbox countMailbox timed out') From cc18cda193d7ad18150eed49f8cea9f37fa16486 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 18:14:54 +0000 Subject: [PATCH 28/62] test(email): cover outbound storage reservation USER_METER shadow handoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a focused workers test that sends a successful outbound email, waits for the best-effort D1→UserMeter storage shadow, and asserts readStorageBytes matches authoritative users.d1_storage_bytes. Co-authored-by: Kent C. Dodds --- .../outbound-storage-shadow.workers.test.ts | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 packages/worker/src/email/outbound-storage-shadow.workers.test.ts diff --git a/packages/worker/src/email/outbound-storage-shadow.workers.test.ts b/packages/worker/src/email/outbound-storage-shadow.workers.test.ts new file mode 100644 index 0000000000..fae92b2a9a --- /dev/null +++ b/packages/worker/src/email/outbound-storage-shadow.workers.test.ts @@ -0,0 +1,127 @@ +import { env } from 'cloudflare:workers' +import { expect, test } from 'vitest' +import { readUserD1StorageBytes } from '#worker/entitlements/service.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { sendOutboundEmail } from './outbound.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const platformBaseUrl = 'https://kody.example.com' +const platformDomain = 'inbox.kody.example.com' + +function createBindingSendEnv() { + return { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: { + async send() { + return { messageId: 'provider-storage-shadow' } + }, + }, + } +} + +async function seedVerifiedAccount(email: string) { + const username = `shadow-${crypto.randomUUID().slice(0, 8)}` + const stableUserId = await createStableUserIdFromEmail(email) + await env.APP_DB.prepare( + `INSERT INTO users (username, email, password_hash, email_verified_at, plan, stable_user_id) + VALUES (?, ?, ?, ?, ?, ?)`, + ) + .bind( + username, + email, + 'test-password-hash', + new Date().toISOString(), + 'max', + stableUserId, + ) + .run() + return { username, from: `${username}@${platformDomain}`, userId: stableUserId } +} + +async function seedAuthoritativeD1StorageBytes(userId: string, bytes: number) { + const updatedAt = new Date().toISOString() + await env.APP_DB.prepare( + `UPDATE users + SET d1_storage_bytes = ?, + d1_storage_bytes_updated_at = ? + WHERE stable_user_id = ?`, + ) + .bind(bytes, updatedAt, userId) + .run() +} + +async function waitForUserMeterStorageShadowParity(input: { + userId: string + timeoutMs?: number +}) { + const meter = userMeterRpc({ env, userId: input.userId }) + const timeoutMs = input.timeoutMs ?? 5_000 + const started = Date.now() + + while (Date.now() - started <= timeoutMs) { + const authoritative = await readUserD1StorageBytes({ + db: env.APP_DB, + userId: input.userId, + }) + const shadow = await meter.readStorageBytes() + if (shadow.outcome === 'ready' && shadow.bytes === authoritative) { + return { authoritative, shadow } + } + await new Promise((resolve) => setTimeout(resolve, 10)) + } + + const authoritative = await readUserD1StorageBytes({ + db: env.APP_DB, + userId: input.userId, + }) + const shadow = await meter.readStorageBytes() + throw new Error( + `Timed out waiting for UserMeter storage shadow parity (D1=${authoritative}, shadow=${shadow.outcome === 'ready' ? shadow.bytes : shadow.outcome}).`, + ) +} + +test('sendOutboundEmail reserves D1 storage bytes and best-effort shadows them into USER_METER', async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + + const accountEmail = `storage-shadow-${crypto.randomUUID()}@example.com` + const { userId } = await seedVerifiedAccount(accountEmail) + const initialD1Bytes = 42 + const staleShadowBytes = 7 + await seedAuthoritativeD1StorageBytes(userId, initialD1Bytes) + await userMeterRpc({ env, userId }).setStorageBytes({ + bytes: staleShadowBytes, + updatedAt: '2026-07-31T11:00:00.000Z', + }) + + const result = await sendOutboundEmail({ + env: createBindingSendEnv(), + userId, + accountEmail, + recipientPolicy: 'self', + subject: 'Storage shadow handoff', + text: 'Outbound body for storage reservation.', + }) + + expect(result.status).toBe('sent') + expect(result.providerMessageId).toBe('provider-storage-shadow') + + const authoritativeAfter = await readUserD1StorageBytes({ + db: env.APP_DB, + userId, + }) + expect(authoritativeAfter).toBeGreaterThan(initialD1Bytes) + + const { authoritative, shadow } = await waitForUserMeterStorageShadowParity({ + userId, + }) + expect(shadow).toMatchObject({ + outcome: 'ready', + bytes: authoritativeAfter, + }) + expect(authoritative).toBe(authoritativeAfter) + expect(authoritative).not.toBe(staleShadowBytes) +}, 30_000) From 2e355c20041a40eee2b5c9e0e3517b14272cc135 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 18:15:14 +0000 Subject: [PATCH 29/62] Add inbound storage reservation UserMeter handoff workers test Exercise handleInboundEmail storage-byte reservation with real UserMeter: capture ExecutionContext waitUntil, drain shadow sync, and assert readStorageBytes matches authoritative users.d1_storage_bytes. Also cover over-quota rejection without meter drift and pre-commit retry without double-reserving D1 storage. Co-authored-by: Kent C. Dodds --- .../inbound-storage-meter.workers.test.ts | 294 ++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 packages/worker/src/email/inbound-storage-meter.workers.test.ts diff --git a/packages/worker/src/email/inbound-storage-meter.workers.test.ts b/packages/worker/src/email/inbound-storage-meter.workers.test.ts new file mode 100644 index 0000000000..fe2dfa6e79 --- /dev/null +++ b/packages/worker/src/email/inbound-storage-meter.workers.test.ts @@ -0,0 +1,294 @@ +import { env } from 'cloudflare:workers' +import { expect, test } from 'vitest' +import { planLimits } from '#worker/entitlements/plans.ts' +import { + estimateEntitlementStorageEntryBytes, + readUserD1StorageBytes, +} from '#worker/entitlements/service.ts' +import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { createWaitUntilDrain } from '#worker/test-support/user-meter.ts' +import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { buildInboundDelivery } from './inbound-delivery.ts' +import { handleInboundEmail } from './inbound.ts' +import { listEmailMessages } from './repo.ts' +import { RetryableInboundStorageError } from './service.ts' +import { createForwardableEmailMessage } from './test-fixtures.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const platformBaseUrl = 'https://kody.example.com' +const platformDomain = 'inbox.kody.example.com' +const inboundStorageMeterTimeoutMs = 30_000 + +function createInboundEnv() { + return { ...env, APP_BASE_URL: platformBaseUrl } +} + +function createCapturedWaitUntilContext() { + const waitUntilPromises: Array> = [] + const ctx = { + waitUntil(promise: Promise) { + waitUntilPromises.push(promise) + }, + passThroughOnException() {}, + } as ExecutionContext + return { ctx, waitUntilPromises } +} + +async function drainWaitUntil(waitUntilPromises: Array>) { + for (let index = 0; index < waitUntilPromises.length; index += 1) { + await waitUntilPromises[index] + } +} + +function estimateInboundEmailStorageBytes(input: { + message: ForwardableEmailMessage + recipient: string +}) { + return ( + input.message.rawSize * 2 + + estimateEntitlementStorageEntryBytes({ + value: { + from: input.message.from, + to: input.message.to, + recipient: input.recipient, + headers: Object.fromEntries(input.message.headers.entries()), + }, + }) + ) +} + +async function seedAccountWithPlan(input: { + email: string + plan: 'free' | 'max' + d1StorageBytes?: number +}) { + const username = `storage-meter-${crypto.randomUUID().slice(0, 8)}` + const stableUserId = await createStableUserIdFromEmail(input.email) + const now = new Date().toISOString() + await env.APP_DB.prepare( + `INSERT INTO users ( + username, email, password_hash, email_verified_at, plan, stable_user_id, + d1_storage_bytes, d1_storage_bytes_updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + username, + input.email, + 'test-password-hash', + now, + input.plan, + stableUserId, + input.d1StorageBytes ?? 0, + now, + ) + .run() + return { username, address: `${username}@${platformDomain}`, stableUserId } +} + +function createFailingEmailBlobs() { + return new Proxy(env.EMAIL_BLOBS, { + get(target, property, receiver) { + if (property === 'put') { + return async () => { + throw new Error('simulated inbound blob put failure') + } + } + const value = Reflect.get(target, property, receiver) + return typeof value === 'function' ? value.bind(target) : value + }, + }) +} + +async function readAuthoritativeD1StorageBytes(userId: string) { + return await readUserD1StorageBytes({ db: env.APP_DB, userId }) +} + +test( + 'inbound storage reservation shadows authoritative D1 bytes into UserMeter via waitUntil', + async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + await ensureUsageRollupsTestSchema(env.APP_DB) + + const initialBytes = 512 + const staleMeterBytes = 999_999 + const email = `storage-meter-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(email) + const { address } = await seedAccountWithPlan({ + email, + plan: 'free', + d1StorageBytes: initialBytes, + }) + const meter = userMeterRpc({ env, userId }) + await meter.setStorageBytes({ + bytes: staleMeterBytes, + updatedAt: '2026-07-31T00:00:00.000Z', + }) + + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Storage meter shadow', + `Message-ID: `, + '', + 'Reserve then shadow.', + ].join('\r\n') + const message = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + const reservedBytes = estimateInboundEmailStorageBytes({ + message, + recipient: address, + }) + const drain = createWaitUntilDrain() + + await handleInboundEmail(message, createInboundEnv(), { + waitUntil: drain.waitUntil, + passThroughOnException() {}, + } as ExecutionContext) + expect(message.rejectedReason).toBeNull() + expect( + await listEmailMessages({ db: env.APP_DB, userId, limit: 10 }), + ).toHaveLength(1) + + const authoritativeAfterReserve = initialBytes + reservedBytes + expect(await readAuthoritativeD1StorageBytes(userId)).toBe( + authoritativeAfterReserve, + ) + + await drain.drain() + expect(await meter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: authoritativeAfterReserve, + }) + expect(await readAuthoritativeD1StorageBytes(userId)).toBe( + authoritativeAfterReserve, + ) + + const storageLimit = planLimits.free.maxStorageBytes + if (storageLimit === null) throw new Error('Expected a numeric free cap.') + const atCapEmail = `storage-meter-cap-${crypto.randomUUID()}@example.com` + const atCapUserId = await createStableUserIdFromEmail(atCapEmail) + const { address: atCapAddress } = await seedAccountWithPlan({ + email: atCapEmail, + plan: 'free', + d1StorageBytes: storageLimit, + }) + const atCapMeter = userMeterRpc({ env, userId: atCapUserId }) + await atCapMeter.setStorageBytes({ + bytes: 123, + updatedAt: '2026-07-31T01:00:00.000Z', + }) + const overQuotaDrain = createWaitUntilDrain() + const overQuotaMessage = createForwardableEmailMessage({ + from: 'sender@example.net', + to: atCapAddress, + raw: [ + 'From: Sender ', + `To: ${atCapAddress}`, + 'Subject: Over storage cap', + `Message-ID: `, + '', + 'Denied.', + ].join('\r\n'), + }) + await handleInboundEmail(overQuotaMessage, createInboundEnv(), { + waitUntil: overQuotaDrain.waitUntil, + passThroughOnException() {}, + } as ExecutionContext) + expect(overQuotaMessage.rejectedReason).toBe( + 'Recipient mailbox is over quota.', + ) + expect(await readAuthoritativeD1StorageBytes(atCapUserId)).toBe(storageLimit) + await overQuotaDrain.drain() + expect(await atCapMeter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: 123, + }) + + const retryEmail = `storage-meter-retry-${crypto.randomUUID()}@example.com` + const retryUserId = await createStableUserIdFromEmail(retryEmail) + const { address: retryAddress } = await seedAccountWithPlan({ + email: retryEmail, + plan: 'max', + d1StorageBytes: 64, + }) + const retryMeter = userMeterRpc({ env, userId: retryUserId }) + const retryRaw = [ + 'From: Sender ', + `To: ${retryAddress}`, + 'Subject: Retry without double reserve', + 'Message-ID: ', + '', + 'Retry body.', + ].join('\r\n') + const retryReservedBytes = estimateInboundEmailStorageBytes({ + message: createForwardableEmailMessage({ + from: 'sender@example.net', + to: retryAddress, + raw: retryRaw, + }), + recipient: retryAddress, + }) + const failingEnv = { + ...createInboundEnv(), + EMAIL_BLOBS: createFailingEmailBlobs(), + } as Parameters[1] + const failCtx = createCapturedWaitUntilContext() + const firstAttempt = createForwardableEmailMessage({ + from: 'sender@example.net', + to: retryAddress, + raw: retryRaw, + }) + await expect( + handleInboundEmail(firstAttempt, failingEnv, failCtx.ctx), + ).rejects.toBeInstanceOf(RetryableInboundStorageError) + await drainWaitUntil(failCtx.waitUntilPromises) + const bytesAfterFailedAttempt = 64 + retryReservedBytes + expect(await readAuthoritativeD1StorageBytes(retryUserId)).toBe( + bytesAfterFailedAttempt, + ) + expect(await retryMeter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: bytesAfterFailedAttempt, + }) + + const retryCtx = createCapturedWaitUntilContext() + const retryAttempt = createForwardableEmailMessage({ + from: 'sender@example.net', + to: retryAddress, + raw: retryRaw, + }) + await handleInboundEmail(retryAttempt, createInboundEnv(), retryCtx.ctx) + await drainWaitUntil(retryCtx.waitUntilPromises) + expect(retryAttempt.rejectedReason).toBeNull() + expect(await readAuthoritativeD1StorageBytes(retryUserId)).toBe( + bytesAfterFailedAttempt, + ) + expect(await retryMeter.readStorageBytes()).toMatchObject({ + outcome: 'ready', + bytes: bytesAfterFailedAttempt, + }) + + const candidate = await buildInboundDelivery({ + userId: retryUserId, + inboxId: 'unused', + recipient: retryAddress, + envelopeFrom: 'sender@example.net', + rawMime: retryRaw, + quotaDay: new Date().toISOString().slice(0, 10), + }) + expect( + await listEmailMessages({ + db: env.APP_DB, + userId: retryUserId, + limit: 1, + }), + ).toMatchObject([{ id: candidate.messageId }]) + }, + inboundStorageMeterTimeoutMs, +) From 55c9550f05b8e230898f0ed8c3880b15409a0bf6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 18:15:42 +0000 Subject: [PATCH 30/62] fix(email): hand storage reservations to UserMeter Co-authored-by: Kent C. Dodds --- packages/worker/src/email/inbound.ts | 4 ++++ packages/worker/src/email/outbound.ts | 1 + 2 files changed, 5 insertions(+) diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index 2e8855eb0a..7cb9a31e86 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -478,12 +478,16 @@ export async function handleInboundEmail( // after unrelated writes fill the mailbox. await assertWithinStorageBytesEntitlement({ db: env.APP_DB, + env, userId, email: account.email, requested: estimateInboundEmailStorageBytes({ message, recipient, }), + waitUntil: ctx + ? (promise: Promise) => ctx.waitUntil(promise) + : undefined, }) await assertWithinEntitlement({ db: env.APP_DB, diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index 5b3c2d0684..c2bd05b91e 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -571,6 +571,7 @@ export async function sendOutboundEmail( }) await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, + env: input.env, userId: input.userId, email: sender.accountEmail, requested: From 0b352445eb746013547db8c58ade8ee9d92b3832 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 18:24:58 +0000 Subject: [PATCH 31/62] fix(email): await outbound storage accounting handoff Co-authored-by: Kent C. Dodds --- .../inbound-storage-meter.workers.test.ts | 4 +- .../outbound-storage-shadow.workers.test.ts | 40 ++++--------------- packages/worker/src/email/outbound.ts | 7 ++++ 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/packages/worker/src/email/inbound-storage-meter.workers.test.ts b/packages/worker/src/email/inbound-storage-meter.workers.test.ts index fe2dfa6e79..f6a7059b4f 100644 --- a/packages/worker/src/email/inbound-storage-meter.workers.test.ts +++ b/packages/worker/src/email/inbound-storage-meter.workers.test.ts @@ -203,7 +203,9 @@ test( expect(overQuotaMessage.rejectedReason).toBe( 'Recipient mailbox is over quota.', ) - expect(await readAuthoritativeD1StorageBytes(atCapUserId)).toBe(storageLimit) + expect(await readAuthoritativeD1StorageBytes(atCapUserId)).toBe( + storageLimit, + ) await overQuotaDrain.drain() expect(await atCapMeter.readStorageBytes()).toMatchObject({ outcome: 'ready', diff --git a/packages/worker/src/email/outbound-storage-shadow.workers.test.ts b/packages/worker/src/email/outbound-storage-shadow.workers.test.ts index fae92b2a9a..a764c4808a 100644 --- a/packages/worker/src/email/outbound-storage-shadow.workers.test.ts +++ b/packages/worker/src/email/outbound-storage-shadow.workers.test.ts @@ -38,7 +38,11 @@ async function seedVerifiedAccount(email: string) { stableUserId, ) .run() - return { username, from: `${username}@${platformDomain}`, userId: stableUserId } + return { + username, + from: `${username}@${platformDomain}`, + userId: stableUserId, + } } async function seedAuthoritativeD1StorageBytes(userId: string, bytes: number) { @@ -53,36 +57,6 @@ async function seedAuthoritativeD1StorageBytes(userId: string, bytes: number) { .run() } -async function waitForUserMeterStorageShadowParity(input: { - userId: string - timeoutMs?: number -}) { - const meter = userMeterRpc({ env, userId: input.userId }) - const timeoutMs = input.timeoutMs ?? 5_000 - const started = Date.now() - - while (Date.now() - started <= timeoutMs) { - const authoritative = await readUserD1StorageBytes({ - db: env.APP_DB, - userId: input.userId, - }) - const shadow = await meter.readStorageBytes() - if (shadow.outcome === 'ready' && shadow.bytes === authoritative) { - return { authoritative, shadow } - } - await new Promise((resolve) => setTimeout(resolve, 10)) - } - - const authoritative = await readUserD1StorageBytes({ - db: env.APP_DB, - userId: input.userId, - }) - const shadow = await meter.readStorageBytes() - throw new Error( - `Timed out waiting for UserMeter storage shadow parity (D1=${authoritative}, shadow=${shadow.outcome === 'ready' ? shadow.bytes : shadow.outcome}).`, - ) -} - test('sendOutboundEmail reserves D1 storage bytes and best-effort shadows them into USER_METER', async () => { silenceIncidentalRuntimeWarnings() await ensureEmailTestSchema(env.APP_DB) @@ -115,9 +89,11 @@ test('sendOutboundEmail reserves D1 storage bytes and best-effort shadows them i }) expect(authoritativeAfter).toBeGreaterThan(initialD1Bytes) - const { authoritative, shadow } = await waitForUserMeterStorageShadowParity({ + const authoritative = await readUserD1StorageBytes({ + db: env.APP_DB, userId, }) + const shadow = await userMeterRpc({ env, userId }).readStorageBytes() expect(shadow).toMatchObject({ outcome: 'ready', bytes: authoritativeAfter, diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index c2bd05b91e..3ab9552545 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -569,6 +569,7 @@ export async function sendOutboundEmail( inReplyTo: input.inReplyToHeader ?? null, references: input.references ?? [], }) + let storageAccounting: Promise | undefined await assertWithinStorageBytesEntitlement({ db: input.env.APP_DB, env: input.env, @@ -587,7 +588,13 @@ export async function sendOutboundEmail( headers: storedHeaders, }, }) + attachmentBytesTotal, + waitUntil: (promise) => { + storageAccounting = promise + }, }) + // Outbound has no ExecutionContext. Await the interface's caught + // accounting task so a worker shutdown cannot drop the UserMeter handoff. + await storageAccounting // Atomic check-and-increment via UserMeter (sole daily authority). await consumeDailyEntitlement({ From 928412de1639f0fbcd7dadb1d147d977b50d4e79 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 18:38:50 +0000 Subject: [PATCH 32/62] fix(email): await fallback inbound storage accounting Co-authored-by: Kent C. Dodds --- .../src/email/inbound-storage-meter.workers.test.ts | 10 +++------- packages/worker/src/email/inbound.ts | 6 +++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/worker/src/email/inbound-storage-meter.workers.test.ts b/packages/worker/src/email/inbound-storage-meter.workers.test.ts index f6a7059b4f..400a22cc50 100644 --- a/packages/worker/src/email/inbound-storage-meter.workers.test.ts +++ b/packages/worker/src/email/inbound-storage-meter.workers.test.ts @@ -144,12 +144,9 @@ test( message, recipient: address, }) - const drain = createWaitUntilDrain() - - await handleInboundEmail(message, createInboundEnv(), { - waitUntil: drain.waitUntil, - passThroughOnException() {}, - } as ExecutionContext) + // Direct/test callers may not have an ExecutionContext. The reservation + // must await its caught UserMeter accounting task before returning. + await handleInboundEmail(message, createInboundEnv()) expect(message.rejectedReason).toBeNull() expect( await listEmailMessages({ db: env.APP_DB, userId, limit: 10 }), @@ -160,7 +157,6 @@ test( authoritativeAfterReserve, ) - await drain.drain() expect(await meter.readStorageBytes()).toMatchObject({ outcome: 'ready', bytes: authoritativeAfterReserve, diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index 7cb9a31e86..a9d1ca09fe 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -472,6 +472,7 @@ export async function handleInboundEmail( }) if (!existingDelivery) { try { + let storageAccounting: Promise | undefined // New deliveries check storage bytes and stored-message caps // before their durable quota claim. A retry with an existing // ledger bypasses both so already-charged mail can still repair @@ -487,8 +488,11 @@ export async function handleInboundEmail( }), waitUntil: ctx ? (promise: Promise) => ctx.waitUntil(promise) - : undefined, + : (promise) => { + storageAccounting = promise + }, }) + if (!ctx) await storageAccounting await assertWithinEntitlement({ db: env.APP_DB, userId, From 75e4f894834a18842f26c8739cb9bfd288b88f95 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 18:45:58 +0000 Subject: [PATCH 33/62] refactor(email): centralize storage accounting lifecycle Co-authored-by: Kent C. Dodds --- packages/worker/src/email/inbound.ts | 10 ++---- packages/worker/src/email/outbound.ts | 11 ++---- .../worker/src/email/storage-reservation.ts | 35 +++++++++++++++++++ 3 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 packages/worker/src/email/storage-reservation.ts diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index a9d1ca09fe..adb4147f6d 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -9,7 +9,6 @@ import { } from '#worker/entitlements/plans.ts' import { assertWithinEntitlement, - assertWithinStorageBytesEntitlement, estimateEntitlementStorageEntryBytes, } from '#worker/entitlements/service.ts' import { recordUsage } from '#worker/usage/record-usage.ts' @@ -62,6 +61,7 @@ import { RetryableInboundStorageError, storeIdempotentInboundEmail, } from './service.ts' +import { reserveEmailStorageBytes } from './storage-reservation.ts' import { countStoredSystemEmailMessages, ensureSystemEmailInbox, @@ -472,12 +472,11 @@ export async function handleInboundEmail( }) if (!existingDelivery) { try { - let storageAccounting: Promise | undefined // New deliveries check storage bytes and stored-message caps // before their durable quota claim. A retry with an existing // ledger bypasses both so already-charged mail can still repair // after unrelated writes fill the mailbox. - await assertWithinStorageBytesEntitlement({ + await reserveEmailStorageBytes({ db: env.APP_DB, env, userId, @@ -488,11 +487,8 @@ export async function handleInboundEmail( }), waitUntil: ctx ? (promise: Promise) => ctx.waitUntil(promise) - : (promise) => { - storageAccounting = promise - }, + : undefined, }) - if (!ctx) await storageAccounting await assertWithinEntitlement({ db: env.APP_DB, userId, diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index 3ab9552545..136e49e4ff 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -11,7 +11,6 @@ import { withAccountWriteLease } from '#worker/account/deletion-state.ts' import { McpCallerError } from '#mcp/caller-error.ts' import { assertWithinEntitlement, - assertWithinStorageBytesEntitlement, consumeDailyEntitlement, estimateEntitlementStorageEntryBytes, } from '#worker/entitlements/service.ts' @@ -27,6 +26,7 @@ import { recordEmailReportingEvent, type EmailReportingEnv, } from './reporting-events.ts' +import { reserveEmailStorageBytes } from './storage-reservation.ts' import { createEmailThread, emailAttachmentBlobKey, @@ -569,8 +569,7 @@ export async function sendOutboundEmail( inReplyTo: input.inReplyToHeader ?? null, references: input.references ?? [], }) - let storageAccounting: Promise | undefined - await assertWithinStorageBytesEntitlement({ + await reserveEmailStorageBytes({ db: input.env.APP_DB, env: input.env, userId: input.userId, @@ -588,13 +587,7 @@ export async function sendOutboundEmail( headers: storedHeaders, }, }) + attachmentBytesTotal, - waitUntil: (promise) => { - storageAccounting = promise - }, }) - // Outbound has no ExecutionContext. Await the interface's caught - // accounting task so a worker shutdown cannot drop the UserMeter handoff. - await storageAccounting // Atomic check-and-increment via UserMeter (sole daily authority). await consumeDailyEntitlement({ diff --git a/packages/worker/src/email/storage-reservation.ts b/packages/worker/src/email/storage-reservation.ts new file mode 100644 index 0000000000..de9ce70b9d --- /dev/null +++ b/packages/worker/src/email/storage-reservation.ts @@ -0,0 +1,35 @@ +import { assertWithinStorageBytesEntitlement } from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' + +/** + * Reserve authoritative D1 storage bytes and durably hand the resulting + * absolute accounting state to UserMeter. + * + * Request handlers should pass their `waitUntil`; direct/service callers + * without an ExecutionContext await the helper's caught accounting task. + */ +export async function reserveEmailStorageBytes(input: { + db: D1Database + env: UserMeterEnv + userId: string + email: string | null | undefined + requested: number + waitUntil?: (promise: Promise) => void +}) { + if (input.waitUntil) { + await assertWithinStorageBytesEntitlement({ + ...input, + waitUntil: input.waitUntil, + }) + return + } + + let accounting: Promise | undefined + await assertWithinStorageBytesEntitlement({ + ...input, + waitUntil: (promise) => { + accounting = promise + }, + }) + await accounting +} From 39eae667435ced05f9c7513d41132883c52db478 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 20:16:26 +0000 Subject: [PATCH 34/62] feat(admin): add Mailbox maintenance controls Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 17 +- .../admin/mailbox-maintenance.node.test.ts | 288 ++++++++++++++++ .../worker/src/admin/mailbox-maintenance.ts | 316 ++++++++++++++++++ packages/worker/src/email/mailbox-do.ts | 39 ++- .../email/mailbox-retention.workers.test.ts | 110 ++++++ packages/worker/src/email/mailbox-types.ts | 18 + .../admin-mailbox-maintenance.node.test.ts | 210 ++++++++++++ .../admin/admin-mailbox-maintenance.ts | 190 +++++++++++ .../src/mcp/capabilities/admin/domain.ts | 7 +- .../mcp/capabilities/registry.node.test.ts | 4 + 10 files changed, 1192 insertions(+), 7 deletions(-) create mode 100644 packages/worker/src/admin/mailbox-maintenance.node.test.ts create mode 100644 packages/worker/src/admin/mailbox-maintenance.ts create mode 100644 packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts create mode 100644 packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index a8edec4bbd..7f465324fe 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -946,7 +946,22 @@ retention pass: successful work with expired rows remaining schedules a near-immediate continuation (`mailboxRetentionContinuationDelayMs`); R2 delete failures use hourly backoff (`mailboxRetentionRetryDelayMs`). Write-path alarm selection never postpones an earlier existing alarm under sustained writes -(near-equal times within skew keep the existing alarm). +(near-equal times within skew keep the existing alarm). `alarm` and the +owner-bound `runRetentionNow({ ownerId })` RPC share one private retention pass +(natural production cutoffs only — no arbitrary cutoff override) and the same +post-pass alarm reschedule; the RPC returns before/after `countMailbox` +aggregates plus `blobDeleteFailures` / `expiredRemaining` (no row ids or +content). + +**Admin accelerated coverage** (`admin_mailbox_maintenance`; +`packages/worker/src/admin/mailbox-maintenance.ts`): audited admin-only +discriminated actions — `status` (aggregate tracked/matching/mismatch/error/ +incomplete/eligible counts plus matching/check timestamps and earliest cutover; +no email content), `reconcile` (bounded `reconcileMailboxParity`, `batch_size` +max 100, then status), and `retention` (D1-discover bounded non-deleting +non-system owners with mail/parity state, call `runRetentionNow` per owner, +aggregate deletes/failures without owner or message ids). No seed or +arbitrary-cutoff surface. Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts new file mode 100644 index 0000000000..6742eff443 --- /dev/null +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -0,0 +1,288 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' +import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' +import { systemEmailOwnerId } from '#worker/email/email-owner.ts' +import { + mailboxReadCutoverCheckedAtMaxAgeMs, + mailboxReadCutoverSoakMs, +} from '#worker/email/mailbox-read-cutover.ts' +import type * as MailboxClientModule from '#worker/email/mailbox-client.ts' +import type * as MailboxReconcileModule from '#worker/email/mailbox-reconcile.ts' + +const mocks = vi.hoisted(() => ({ + reconcileMailboxParity: vi.fn(), + mailboxRpc: vi.fn(), +})) + +vi.mock('#worker/email/mailbox-reconcile.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + reconcileMailboxParity: mocks.reconcileMailboxParity, + } +}) + +vi.mock('#worker/email/mailbox-client.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + mailboxRpc: mocks.mailboxRpc, + } +}) + +const { + adminMailboxMaintenanceMaxBatchSize, + loadAdminMailboxMaintenanceStatus, + runAdminMailboxMaintenanceReconcile, + runAdminMailboxMaintenanceRetention, +} = await import('./mailbox-maintenance.ts') + +const now = new Date('2026-08-01T12:00:00.000Z') + +function createMaintenanceDb() { + const sqlite = new DatabaseSync(':memory:') + sqlite.exec(` + CREATE TABLE users ( + id INTEGER PRIMARY KEY, + stable_user_id TEXT UNIQUE NOT NULL, + username TEXT NOT NULL, + email TEXT NOT NULL, + deleting_at TEXT, + mailbox_parity_checked_at TEXT, + mailbox_parity_matching_since TEXT, + mailbox_parity_mismatch_count INTEGER NOT NULL DEFAULT 0, + mailbox_parity_last_error TEXT, + mailbox_parity_content_watermark_at TEXT, + mailbox_parity_content_replay_upper_at TEXT, + mailbox_parity_content_replay_cursor_id TEXT, + mailbox_parity_message_backfill_cursor_id TEXT, + mailbox_parity_message_backfill_completed_at TEXT, + mailbox_parity_event_backfill_cursor_id TEXT, + mailbox_parity_event_backfill_completed_at TEXT, + created_at TEXT NOT NULL DEFAULT '2026-01-01T00:00:00.000Z', + updated_at TEXT NOT NULL DEFAULT '2026-01-01T00:00:00.000Z' + ); + CREATE TABLE email_messages ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL + ); + CREATE TABLE email_delivery_events ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL + ); + `) + return { sqlite, db: createD1FromSqlite(sqlite) } +} + +function insertTrackedUser( + sqlite: DatabaseSync, + input: { + email: string + checkedAt?: string | null + matchingSince?: string | null + mismatchCount?: number + lastError?: string | null + messagesCompletedAt?: string | null + eventsCompletedAt?: string | null + contentReplayUpperAt?: string | null + deletingAt?: string | null + stableUserId?: string + }, +) { + const stableUserId = + input.stableUserId ?? testStableUserIdFromEmail(input.email) + sqlite + .prepare( + `INSERT INTO users ( + stable_user_id, username, email, deleting_at, + mailbox_parity_checked_at, mailbox_parity_matching_since, + mailbox_parity_mismatch_count, mailbox_parity_last_error, + mailbox_parity_message_backfill_completed_at, + mailbox_parity_event_backfill_completed_at, + mailbox_parity_content_replay_upper_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + stableUserId, + input.email.split('@')[0] ?? 'user', + input.email, + input.deletingAt ?? null, + input.checkedAt ?? null, + input.matchingSince ?? null, + input.mismatchCount ?? 0, + input.lastError ?? null, + input.messagesCompletedAt ?? null, + input.eventsCompletedAt ?? null, + input.contentReplayUpperAt ?? null, + ) + return stableUserId +} + +test('loadAdminMailboxMaintenanceStatus aggregates buckets without owner ids', async () => { + const { sqlite, db } = createMaintenanceDb() + const matchingSince = new Date( + now.getTime() - mailboxReadCutoverSoakMs - 60_000, + ).toISOString() + const freshCheckedAt = new Date( + now.getTime() - mailboxReadCutoverCheckedAtMaxAgeMs / 2, + ).toISOString() + const incompleteCheckedAt = '2026-07-01T00:00:00.000Z' + + insertTrackedUser(sqlite, { + email: 'matching@example.com', + matchingSince, + checkedAt: freshCheckedAt, + messagesCompletedAt: matchingSince, + eventsCompletedAt: matchingSince, + }) + insertTrackedUser(sqlite, { + email: 'mismatch@example.com', + checkedAt: incompleteCheckedAt, + mismatchCount: 2, + }) + insertTrackedUser(sqlite, { + email: 'error@example.com', + checkedAt: incompleteCheckedAt, + lastError: 'countMailbox timed out', + }) + insertTrackedUser(sqlite, { + email: 'incomplete@example.com', + checkedAt: incompleteCheckedAt, + messagesCompletedAt: null, + }) + insertTrackedUser(sqlite, { + email: 'deleting@example.com', + checkedAt: freshCheckedAt, + matchingSince, + deletingAt: now.toISOString(), + messagesCompletedAt: matchingSince, + eventsCompletedAt: matchingSince, + }) + insertTrackedUser(sqlite, { + email: 'system@example.com', + stableUserId: systemEmailOwnerId, + checkedAt: freshCheckedAt, + matchingSince, + messagesCompletedAt: matchingSince, + eventsCompletedAt: matchingSince, + }) + + const status = await loadAdminMailboxMaintenanceStatus({ db, now }) + expect(status).toMatchObject({ + generatedAt: now.toISOString(), + trackedOwners: 4, + matching: 1, + mismatch: 1, + error: 1, + incomplete: 1, + eligible: 1, + oldestMatchingSince: matchingSince, + newestMatchingSince: matchingSince, + earliestCutoverAt: new Date( + Date.parse(matchingSince) + mailboxReadCutoverSoakMs, + ).toISOString(), + }) + const serialized = JSON.stringify(status) + expect(serialized).not.toContain('matching@example.com') + expect(serialized).not.toContain(systemEmailOwnerId) + expect(serialized).not.toContain('countMailbox timed out') +}) + +test('runAdminMailboxMaintenanceReconcile clamps batch_size and returns status', async () => { + const { db } = createMaintenanceDb() + mocks.reconcileMailboxParity.mockResolvedValue({ + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + const env = { APP_DB: db, MAILBOX: {} } as unknown as Env + const result = await runAdminMailboxMaintenanceReconcile({ + env, + batchSize: 10_000, + now, + }) + expect(mocks.reconcileMailboxParity).toHaveBeenCalledWith({ + env, + now, + batchSize: adminMailboxMaintenanceMaxBatchSize, + }) + expect(result.metrics.matched).toBe(1) + expect(result.status.trackedOwners).toBe(0) +}) + +test('runAdminMailboxMaintenanceRetention aggregates owner results without ids', async () => { + consoleWarn.mockImplementation(() => {}) + const { sqlite, db } = createMaintenanceDb() + const ownerA = insertTrackedUser(sqlite, { + email: 'ret-a@example.com', + checkedAt: '2026-07-01T00:00:00.000Z', + }) + const ownerB = insertTrackedUser(sqlite, { + email: 'ret-b@example.com', + checkedAt: '2026-07-02T00:00:00.000Z', + }) + sqlite + .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) + .run('m1', ownerA) + sqlite + .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) + .run('m2', ownerB) + + const runRetentionNow = vi + .fn() + .mockResolvedValueOnce({ + before: { + threads: 1, + messages: 2, + attachments: 1, + deliveryEvents: 3, + }, + after: { + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + blobDeleteFailures: true, + expiredRemaining: true, + }) + .mockRejectedValueOnce(new Error('mailbox unavailable')) + + mocks.mailboxRpc.mockImplementation(({ userId }: { userId: string }) => { + expect([ownerA, ownerB]).toContain(userId) + return { runRetentionNow } + }) + + const result = await runAdminMailboxMaintenanceRetention({ + env: { APP_DB: db, MAILBOX: {} } as unknown as Env, + batchSize: 10, + now, + }) + + expect(runRetentionNow).toHaveBeenCalledTimes(2) + expect(runRetentionNow).toHaveBeenCalledWith({ ownerId: ownerA }) + expect(result.metrics).toEqual({ + ownersAttempted: 2, + ownersSucceeded: 1, + ownersFailed: 1, + messagesDeleted: 1, + threadsDeleted: 1, + attachmentsDeleted: 1, + deliveryEventsDeleted: 2, + blobDeleteFailureOwners: 1, + expiredRemainingOwners: 1, + }) + const serialized = JSON.stringify(result) + expect(serialized).not.toContain(ownerA) + expect(serialized).not.toContain(ownerB) + expect(serialized).not.toContain('mailbox unavailable') + expect(consoleWarn).toHaveBeenCalledWith( + 'admin-mailbox-maintenance-retention-owner-failed', + expect.objectContaining({ error: expect.any(Error) }), + ) +}) diff --git a/packages/worker/src/admin/mailbox-maintenance.ts b/packages/worker/src/admin/mailbox-maintenance.ts new file mode 100644 index 0000000000..e180d01d43 --- /dev/null +++ b/packages/worker/src/admin/mailbox-maintenance.ts @@ -0,0 +1,316 @@ +import { systemEmailOwnerId } from '#worker/email/email-owner.ts' +import { mailboxRpc, type MailboxEnv } from '#worker/email/mailbox-client.ts' +import { + mailboxParityUserBatchSize, + reconcileMailboxParity, + type MailboxParityReconcileEnv, + type MailboxParityReconcileMetrics, +} from '#worker/email/mailbox-reconcile.ts' +import { listUsersForMailboxParity } from '#worker/email/mailbox-parity-repo.ts' +import { + mailboxReadCutoverCheckedAtMaxAgeMs, + mailboxReadCutoverSoakMs, +} from '#worker/email/mailbox-read-cutover.ts' + +/** Cap for admin reconcile/retention batch discovery (hard max). */ +export const adminMailboxMaintenanceMaxBatchSize = 100 + +export type AdminMailboxMaintenanceEnv = MailboxParityReconcileEnv & MailboxEnv + +export type AdminMailboxMaintenanceStatus = { + generatedAt: string + trackedOwners: number + matching: number + mismatch: number + error: number + incomplete: number + eligible: number + oldestMatchingSince: string | null + newestMatchingSince: string | null + oldestCheckedAt: string | null + newestCheckedAt: string | null + /** Earliest `matching_since + soak` among matching owners (null if none). */ + earliestCutoverAt: string | null +} + +export type AdminMailboxMaintenanceRetentionMetrics = { + ownersAttempted: number + ownersSucceeded: number + ownersFailed: number + messagesDeleted: number + threadsDeleted: number + attachmentsDeleted: number + deliveryEventsDeleted: number + blobDeleteFailureOwners: number + expiredRemainingOwners: number +} + +const trackedOwnerSql = ` + u.deleting_at IS NULL + AND u.stable_user_id != ? + AND ( + EXISTS ( + SELECT 1 FROM email_messages m + WHERE m.user_id = u.stable_user_id + ) + OR EXISTS ( + SELECT 1 FROM email_delivery_events e + WHERE e.user_id = u.stable_user_id + ) + OR u.mailbox_parity_checked_at IS NOT NULL + OR u.mailbox_parity_matching_since IS NOT NULL + OR u.mailbox_parity_last_error IS NOT NULL + OR u.mailbox_parity_mismatch_count > 0 + OR u.mailbox_parity_content_watermark_at IS NOT NULL + OR u.mailbox_parity_content_replay_upper_at IS NOT NULL + OR u.mailbox_parity_content_replay_cursor_id IS NOT NULL + OR u.mailbox_parity_message_backfill_cursor_id IS NOT NULL + OR u.mailbox_parity_message_backfill_completed_at IS NOT NULL + OR u.mailbox_parity_event_backfill_cursor_id IS NOT NULL + OR u.mailbox_parity_event_backfill_completed_at IS NOT NULL + ) +` + +function clampBatchSize(batchSize: number | undefined): number { + const requested = batchSize ?? mailboxParityUserBatchSize + if (!Number.isFinite(requested)) return mailboxParityUserBatchSize + return Math.max( + 1, + Math.min(adminMailboxMaintenanceMaxBatchSize, Math.trunc(requested)), + ) +} + +function countDelta(before: number, after: number): number { + return Math.max(0, before - after) +} + +/** + * Aggregate-only Mailbox parity status for operators. Never returns email + * content, message ids, or per-owner identity. + */ +export async function loadAdminMailboxMaintenanceStatus(input: { + db: D1Database + now?: Date +}): Promise { + const now = input.now ?? new Date() + const generatedAt = now.toISOString() + const nowMs = now.getTime() + const soakCutoffIso = new Date(nowMs - mailboxReadCutoverSoakMs).toISOString() + const checkedAtMinIso = new Date( + nowMs - mailboxReadCutoverCheckedAtMaxAgeMs, + ).toISOString() + + const row = await input.db + .prepare( + `SELECT + COUNT(*) AS trackedOwners, + SUM( + CASE + WHEN u.mailbox_parity_last_error IS NOT NULL THEN 1 + ELSE 0 + END + ) AS errorCount, + SUM( + CASE + WHEN u.mailbox_parity_last_error IS NULL + AND u.mailbox_parity_mismatch_count > 0 + THEN 1 + ELSE 0 + END + ) AS mismatchCount, + SUM( + CASE + WHEN u.mailbox_parity_last_error IS NULL + AND u.mailbox_parity_mismatch_count = 0 + AND u.mailbox_parity_matching_since IS NOT NULL + AND u.mailbox_parity_message_backfill_completed_at IS NOT NULL + AND u.mailbox_parity_event_backfill_completed_at IS NOT NULL + AND u.mailbox_parity_content_replay_upper_at IS NULL + THEN 1 + ELSE 0 + END + ) AS matchingCount, + SUM( + CASE + WHEN u.mailbox_parity_last_error IS NULL + AND u.mailbox_parity_mismatch_count = 0 + AND ( + u.mailbox_parity_matching_since IS NULL + OR u.mailbox_parity_message_backfill_completed_at IS NULL + OR u.mailbox_parity_event_backfill_completed_at IS NULL + OR u.mailbox_parity_content_replay_upper_at IS NOT NULL + ) + THEN 1 + ELSE 0 + END + ) AS incompleteCount, + SUM( + CASE + WHEN u.mailbox_parity_last_error IS NULL + AND u.mailbox_parity_mismatch_count = 0 + AND u.mailbox_parity_matching_since IS NOT NULL + AND u.mailbox_parity_matching_since <= ? + AND u.mailbox_parity_matching_since <= ? + AND u.mailbox_parity_checked_at IS NOT NULL + AND u.mailbox_parity_checked_at >= ? + AND u.mailbox_parity_checked_at <= ? + THEN 1 + ELSE 0 + END + ) AS eligibleCount, + MIN(u.mailbox_parity_matching_since) AS oldestMatchingSince, + MAX(u.mailbox_parity_matching_since) AS newestMatchingSince, + MIN(u.mailbox_parity_checked_at) AS oldestCheckedAt, + MAX(u.mailbox_parity_checked_at) AS newestCheckedAt, + MIN( + CASE + WHEN u.mailbox_parity_last_error IS NULL + AND u.mailbox_parity_mismatch_count = 0 + AND u.mailbox_parity_matching_since IS NOT NULL + THEN u.mailbox_parity_matching_since + ELSE NULL + END + ) AS earliestMatchingSinceForCutover + FROM users u + WHERE ${trackedOwnerSql}`, + ) + .bind( + soakCutoffIso, + generatedAt, + checkedAtMinIso, + generatedAt, + systemEmailOwnerId, + ) + .first<{ + trackedOwners: number + errorCount: number | null + mismatchCount: number | null + matchingCount: number | null + incompleteCount: number | null + eligibleCount: number | null + oldestMatchingSince: string | null + newestMatchingSince: string | null + oldestCheckedAt: string | null + newestCheckedAt: string | null + earliestMatchingSinceForCutover: string | null + }>() + + const earliestMatchingSince = row?.earliestMatchingSinceForCutover ?? null + let earliestCutoverAt: string | null = null + if (earliestMatchingSince != null) { + const matchingMs = Date.parse(earliestMatchingSince) + if (Number.isFinite(matchingMs)) { + earliestCutoverAt = new Date( + matchingMs + mailboxReadCutoverSoakMs, + ).toISOString() + } + } + + return { + generatedAt, + trackedOwners: Number(row?.trackedOwners ?? 0) || 0, + matching: Number(row?.matchingCount ?? 0) || 0, + mismatch: Number(row?.mismatchCount ?? 0) || 0, + error: Number(row?.errorCount ?? 0) || 0, + incomplete: Number(row?.incompleteCount ?? 0) || 0, + eligible: Number(row?.eligibleCount ?? 0) || 0, + oldestMatchingSince: row?.oldestMatchingSince ?? null, + newestMatchingSince: row?.newestMatchingSince ?? null, + oldestCheckedAt: row?.oldestCheckedAt ?? null, + newestCheckedAt: row?.newestCheckedAt ?? null, + earliestCutoverAt, + } +} + +export async function runAdminMailboxMaintenanceReconcile(input: { + env: AdminMailboxMaintenanceEnv + batchSize?: number + now?: Date +}): Promise<{ + metrics: MailboxParityReconcileMetrics + status: AdminMailboxMaintenanceStatus +}> { + const now = input.now ?? new Date() + const metrics = await reconcileMailboxParity({ + env: input.env, + now, + batchSize: clampBatchSize(input.batchSize), + }) + const status = await loadAdminMailboxMaintenanceStatus({ + db: input.env.APP_DB, + now, + }) + return { metrics, status } +} + +/** + * Bounded D1-discovered retention sweep. Calls owner-bound + * `Mailbox.runRetentionNow` (natural cutoffs only). Aggregates counts without + * owner ids, message ids, or email content. + */ +export async function runAdminMailboxMaintenanceRetention(input: { + env: AdminMailboxMaintenanceEnv + batchSize?: number + now?: Date +}): Promise<{ + metrics: AdminMailboxMaintenanceRetentionMetrics + status: AdminMailboxMaintenanceStatus +}> { + const now = input.now ?? new Date() + const owners = await listUsersForMailboxParity({ + db: input.env.APP_DB, + limit: clampBatchSize(input.batchSize), + }) + + const metrics: AdminMailboxMaintenanceRetentionMetrics = { + ownersAttempted: 0, + ownersSucceeded: 0, + ownersFailed: 0, + messagesDeleted: 0, + threadsDeleted: 0, + attachmentsDeleted: 0, + deliveryEventsDeleted: 0, + blobDeleteFailureOwners: 0, + expiredRemainingOwners: 0, + } + + for (const { userId } of owners) { + metrics.ownersAttempted += 1 + try { + const result = await mailboxRpc({ + env: input.env, + userId, + }).runRetentionNow({ ownerId: userId }) + metrics.ownersSucceeded += 1 + metrics.messagesDeleted += countDelta( + result.before.messages, + result.after.messages, + ) + metrics.threadsDeleted += countDelta( + result.before.threads, + result.after.threads, + ) + metrics.attachmentsDeleted += countDelta( + result.before.attachments, + result.after.attachments, + ) + metrics.deliveryEventsDeleted += countDelta( + result.before.deliveryEvents, + result.after.deliveryEvents, + ) + if (result.blobDeleteFailures) metrics.blobDeleteFailureOwners += 1 + if (result.expiredRemaining) metrics.expiredRemainingOwners += 1 + } catch (error) { + metrics.ownersFailed += 1 + console.warn('admin-mailbox-maintenance-retention-owner-failed', { + error, + }) + } + } + + const status = await loadAdminMailboxMaintenanceStatus({ + db: input.env.APP_DB, + now, + }) + return { metrics, status } +} diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 750ba38f96..117e9ee2fd 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -40,6 +40,7 @@ import { type MailboxMessageRecord, type MailboxPartialMutationResult, type MailboxRpc, + type MailboxRunRetentionNowResult, type MailboxSearchMessagesInput, type MailboxSetMessageClassificationInput, type MailboxThreadInput, @@ -138,12 +139,18 @@ class MailboxBase extends DurableObject implements MailboxRpc { this.retentionAlarmArmed = false } - async alarm(): Promise { + /** + * Shared retention pass for `alarm` and {@link runRetentionNow}. + * Natural production cutoffs only; exact post-pass alarm scheduling. + */ + private async runRetentionPass(): Promise { + const before = this.store.countMailbox() const nowMs = Date.now() const result = await enforceMailboxRetention({ store: this.store, blobs: this.env.EMAIL_BLOBS, }) + const after = this.store.countMailbox() const nextDueAtMs = nextMailboxRetentionDueAtMs(this.store) const reschedule = computeMailboxRetentionReschedule({ nowMs, @@ -154,11 +161,32 @@ class MailboxBase extends DurableObject implements MailboxRpc { if (reschedule.atMs == null) { this.retentionAlarmArmed = false this.retentionIdleConfirmed = true - return + } else { + await this.ctx.storage.setAlarm(reschedule.atMs) + this.retentionAlarmArmed = true + this.retentionIdleConfirmed = false } - await this.ctx.storage.setAlarm(reschedule.atMs) - this.retentionAlarmArmed = true - this.retentionIdleConfirmed = false + return { + before, + after, + blobDeleteFailures: result.hadBlobDeleteFailures, + expiredRemaining: result.expiredWorkRemaining, + } + } + + async alarm(): Promise { + await this.runRetentionPass() + } + + /** + * Owner-bound retention pass (natural cutoffs only). Same scheduling as + * `alarm`; returns before/after counts with no row ids or content. + */ + async runRetentionNow(input: { + ownerId: string + }): Promise { + this.store.assertOwner(input.ownerId) + return this.runRetentionPass() } /** Atomic mirror of thread + message + attachments for dual-write. */ @@ -485,6 +513,7 @@ export { type MailboxMessageInput, type MailboxMessageRecord, type MailboxPartialMutationResult, + type MailboxRunRetentionNowResult, type MailboxSetMessageClassificationInput, type MailboxThreadInput, type MailboxThreadRecord, diff --git a/packages/worker/src/email/mailbox-retention.workers.test.ts b/packages/worker/src/email/mailbox-retention.workers.test.ts index 86dc6e4da7..8411864f7b 100644 --- a/packages/worker/src/email/mailbox-retention.workers.test.ts +++ b/packages/worker/src/email/mailbox-retention.workers.test.ts @@ -15,6 +15,7 @@ import { } from './mailbox-do.ts' import { mailboxRetentionAlarmSkewMs } from './mailbox-types.ts' import { + assertMailboxThrows, baseAttachment, baseDeliveryEvent, baseMessage, @@ -277,3 +278,112 @@ test('Mailbox retention deletes canonical R2 keys before metadata and backs off expect(await env.EMAIL_BLOBS.get(failRawKey)).toBeNull() expect(await mailbox.getMessage({ messageId: failMessage.id })).toBeNull() }) + +test('Mailbox runRetentionNow is owner-bound, R2-before-row, and no-ops fresh rows', async () => { + silenceIncidentalRuntimeWarnings() + consoleWarn.mockImplementation((...args: Array) => { + const message = String(args[0] ?? '') + if (message.includes('mailbox-retention-blob-delete-failed')) return + }) + + const userId = uniqueUserId('retention-now') + const otherOwner = uniqueUserId('retention-other') + const mailbox = rpcFor(userId) + const stub = stubFor(userId) + + const oldMessageAt = new Date( + Date.now() - (mailboxMessageRetentionDays + 3) * 24 * 60 * 60 * 1000, + ).toISOString() + const freshAt = new Date().toISOString() + + const keepMessage = baseMessage(userId, { + id: 'keep-fresh', + subject: 'fresh', + createdAt: freshAt, + updatedAt: freshAt, + }) + const dropMessage = baseMessage(userId, { + id: 'drop-old', + threadId: 'drop-thread', + subject: 'old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + const failMessage = baseMessage(userId, { + id: 'fail-old', + subject: 'fail-old', + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }) + + await mailbox.mirrorMessage({ ownerId: userId, message: keepMessage }) + await mailbox.mirrorMessage({ + ownerId: userId, + thread: baseThread({ + id: 'drop-thread', + lastMessageAt: oldMessageAt, + createdAt: oldMessageAt, + updatedAt: oldMessageAt, + }), + message: dropMessage, + }) + await mailbox.mirrorMessage({ ownerId: userId, message: failMessage }) + + await runInDurableObject(stub, async (instance: Mailbox) => { + await assertMailboxThrows(/ownerId mismatch/, () => + instance.runRetentionNow({ ownerId: otherOwner }), + ) + }) + + const dropRawKey = emailRawMimeKey(userId, dropMessage.id) + const failRawKey = emailRawMimeKey(userId, failMessage.id) + const keepRawKey = emailRawMimeKey(userId, keepMessage.id) + await env.EMAIL_BLOBS.put(dropRawKey, 'drop-raw') + await env.EMAIL_BLOBS.put(failRawKey, 'fail-raw') + await env.EMAIL_BLOBS.put(keepRawKey, 'keep-raw') + + const originalDelete = env.EMAIL_BLOBS.delete.bind(env.EMAIL_BLOBS) + const deletedKeys: Array = [] + env.EMAIL_BLOBS.delete = (async (keys: string | Array) => { + const list = Array.isArray(keys) ? keys : [keys] + deletedKeys.push(...list) + if (list.includes(failRawKey)) { + throw new Error('simulated R2 delete failure') + } + return await originalDelete(keys) + }) as typeof env.EMAIL_BLOBS.delete + + try { + const result = await mailbox.runRetentionNow({ ownerId: userId }) + expect(result.before.messages).toBe(3) + expect(result.after.messages).toBe(2) + expect(result.blobDeleteFailures).toBe(true) + expect(result.expiredRemaining).toBe(true) + expect(deletedKeys).toEqual( + expect.arrayContaining([dropRawKey, failRawKey]), + ) + // R2-before-row via existing fake: successful blob delete drops the + // row; failed blob delete retains the row and the R2 object. + expect(await env.EMAIL_BLOBS.get(dropRawKey)).toBeNull() + expect(await env.EMAIL_BLOBS.get(failRawKey)).not.toBeNull() + expect(await env.EMAIL_BLOBS.get(keepRawKey)).not.toBeNull() + expect(await mailbox.getMessage({ messageId: dropMessage.id })).toBeNull() + expect( + await mailbox.getMessage({ messageId: failMessage.id }), + ).toMatchObject({ id: failMessage.id }) + expect( + await mailbox.getMessage({ messageId: keepMessage.id }), + ).toMatchObject({ id: keepMessage.id }) + } finally { + env.EMAIL_BLOBS.delete = originalDelete + } + + // Natural cutoff no-op: only fresh rows remain after retry clears the + // failed delete — later pass reports no further deletions. + await mailbox.runRetentionNow({ ownerId: userId }) + const noop = await mailbox.runRetentionNow({ ownerId: userId }) + expect(noop.before).toEqual(noop.after) + expect(noop.blobDeleteFailures).toBe(false) + expect(noop.expiredRemaining).toBe(false) + expect(noop.after.messages).toBe(1) +}) diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 5999409de4..2cbabe3466 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -325,6 +325,17 @@ export type MailboxCountResult = { deliveryEvents: number } +/** + * Aggregate result from {@link MailboxRpc.runRetentionNow} (and the shared + * private retention pass used by `alarm`). Counts only — no row ids or content. + */ +export type MailboxRunRetentionNowResult = { + before: MailboxCountResult + after: MailboxCountResult + blobDeleteFailures: boolean + expiredRemaining: boolean +} + export type MailboxListMessagesInput = { inboxId?: string | null direction?: EmailDirection | null @@ -539,6 +550,13 @@ export type MailboxRpc = { pageSize?: number startAfter?: string | null }) => Promise + /** + * Owner-bound retention pass using natural production cutoffs only (no + * cutoff override). Reuses the same pass + alarm reschedule as `alarm`. + */ + runRetentionNow: (input: { + ownerId: string + }) => Promise purge: () => Promise<{ ok: true }> } diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts new file mode 100644 index 0000000000..c949ddeb26 --- /dev/null +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts @@ -0,0 +1,210 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import type * as AuditLog from '#worker/audit-log.ts' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { createMcpCallerContext } from '#mcp/context.ts' +import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' +import type * as MailboxMaintenance from '#worker/admin/mailbox-maintenance.ts' + +const mockModule = vi.hoisted(() => ({ + logAuditEvent: vi.fn(async () => undefined), + loadAdminMailboxMaintenanceStatus: vi.fn(), + runAdminMailboxMaintenanceReconcile: vi.fn(), + runAdminMailboxMaintenanceRetention: vi.fn(), +})) + +vi.mock('#worker/audit-log.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + logAuditEvent: (...args: Array) => + mockModule.logAuditEvent(...args), + } +}) + +vi.mock('#worker/admin/mailbox-maintenance.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + loadAdminMailboxMaintenanceStatus: + mockModule.loadAdminMailboxMaintenanceStatus, + runAdminMailboxMaintenanceReconcile: + mockModule.runAdminMailboxMaintenanceReconcile, + runAdminMailboxMaintenanceRetention: + mockModule.runAdminMailboxMaintenanceRetention, + } +}) + +const { adminDomain } = await import('./domain.ts') +const { adminMailboxMaintenanceCapability } = + await import('./admin-mailbox-maintenance.ts') + +const emptyStatus = { + generatedAt: '2026-08-01T12:00:00.000Z', + trackedOwners: 2, + matching: 1, + mismatch: 0, + error: 0, + incomplete: 1, + eligible: 0, + oldestMatchingSince: '2026-07-30T00:00:00.000Z', + newestMatchingSince: '2026-07-30T00:00:00.000Z', + oldestCheckedAt: '2026-07-31T00:00:00.000Z', + newestCheckedAt: '2026-08-01T11:00:00.000Z', + earliestCutoverAt: '2026-07-31T00:00:00.000Z', +} + +function createAdminCtx() { + const sqlite = new DatabaseSync(':memory:') + sqlite.exec(` + CREATE TABLE users ( + id INTEGER PRIMARY KEY, + stable_user_id TEXT UNIQUE NOT NULL, + username TEXT NOT NULL, + email TEXT NOT NULL + ); + CREATE TABLE audit_events ( + id INTEGER PRIMARY KEY, + category TEXT NOT NULL, + action TEXT NOT NULL, + result TEXT NOT NULL, + email_hash TEXT, + ip_hash TEXT, + client_id TEXT, + path TEXT, + reason TEXT, + timestamp TEXT NOT NULL + ); + `) + const db = createD1FromSqlite(sqlite) + return { + env: { APP_DB: db } as Env, + callerContext: createMcpCallerContext({ + baseUrl: 'https://heykody.dev', + user: { + userId: testStableUserIdFromEmail('admin@example.com'), + email: 'admin@example.com', + displayName: 'Admin', + roles: ['admin'], + }, + }), + } +} + +test('admin_mailbox_maintenance is registered admin-only destructive with keywords', () => { + const capability = adminDomain.capabilities.find( + (entry) => entry.name === 'admin_mailbox_maintenance', + ) + expect(capability).toMatchObject({ + requiredRole: 'admin', + readOnly: false, + idempotent: false, + destructive: true, + }) + expect(adminDomain.keywords).toEqual( + expect.arrayContaining(['mailbox', 'maintenance', 'retention']), + ) + expect(adminMailboxMaintenanceCapability.name).toBe( + 'admin_mailbox_maintenance', + ) +}) + +test('admin_mailbox_maintenance status is audited and aggregate-only', async () => { + mockModule.loadAdminMailboxMaintenanceStatus.mockResolvedValue(emptyStatus) + const ctx = createAdminCtx() + const result = await adminMailboxMaintenanceCapability.handler( + { action: 'status' }, + ctx, + ) + expect(result).toEqual({ action: 'status', status: emptyStatus }) + expect(mockModule.logAuditEvent).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'admin_mailbox_maintenance', + result: 'success', + reason: 'action=status;tracked=2', + }), + ) + const serialized = JSON.stringify(result) + expect(serialized).not.toContain('subject') + expect(serialized).not.toContain('textBody') +}) + +test('admin_mailbox_maintenance reconcile and retention route with schema bounds', async () => { + mockModule.runAdminMailboxMaintenanceReconcile.mockResolvedValue({ + metrics: { + scanned: 1, + backfilled: 0, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }, + status: emptyStatus, + }) + mockModule.runAdminMailboxMaintenanceRetention.mockResolvedValue({ + metrics: { + ownersAttempted: 1, + ownersSucceeded: 1, + ownersFailed: 0, + messagesDeleted: 0, + threadsDeleted: 0, + attachmentsDeleted: 0, + deliveryEventsDeleted: 0, + blobDeleteFailureOwners: 0, + expiredRemainingOwners: 0, + }, + status: emptyStatus, + }) + const ctx = createAdminCtx() + + const reconcile = await adminMailboxMaintenanceCapability.handler( + { action: 'reconcile', batch_size: 16 }, + ctx, + ) + expect(reconcile.action).toBe('reconcile') + expect(mockModule.runAdminMailboxMaintenanceReconcile).toHaveBeenCalledWith({ + env: ctx.env, + batchSize: 16, + }) + expect(mockModule.logAuditEvent).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'admin_mailbox_maintenance', + reason: 'action=reconcile;scanned=1;matched=1', + }), + ) + + const retention = await adminMailboxMaintenanceCapability.handler( + { action: 'retention', batch_size: 8 }, + ctx, + ) + expect(retention.action).toBe('retention') + expect(mockModule.runAdminMailboxMaintenanceRetention).toHaveBeenCalledWith({ + env: ctx.env, + batchSize: 8, + }) + expect(mockModule.logAuditEvent).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'admin_mailbox_maintenance', + reason: 'action=retention;attempted=1;succeeded=1', + }), + ) + + await expect( + adminMailboxMaintenanceCapability.handler( + { action: 'reconcile', batch_size: 101 }, + ctx, + ), + ).rejects.toThrow() + await expect( + adminMailboxMaintenanceCapability.handler({ action: 'seed' }, ctx), + ).rejects.toThrow() + await expect( + adminMailboxMaintenanceCapability.handler( + { + action: 'retention', + cutoff: '2000-01-01T00:00:00.000Z', + }, + ctx, + ), + ).rejects.toThrow() +}) diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts new file mode 100644 index 0000000000..d943b03849 --- /dev/null +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts @@ -0,0 +1,190 @@ +import { z } from 'zod' +import { + adminMailboxMaintenanceMaxBatchSize, + loadAdminMailboxMaintenanceStatus, + runAdminMailboxMaintenanceReconcile, + runAdminMailboxMaintenanceRetention, +} from '#worker/admin/mailbox-maintenance.ts' +import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' +import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' +import { + adminMutationCapabilityAccess, + auditAdminCapabilityInvocation, +} from './admin-shared.ts' + +const batchSizeSchema = z + .number() + .int() + .min(1) + .max(adminMailboxMaintenanceMaxBatchSize) + .optional() + .describe( + `Optional bounded owner batch size (1–${adminMailboxMaintenanceMaxBatchSize}). Defaults to the scheduled parity lane batch size.`, + ) + +const statusSchema = z.object({ + generatedAt: z.string(), + trackedOwners: z.number().int().nonnegative(), + matching: z.number().int().nonnegative(), + mismatch: z.number().int().nonnegative(), + error: z.number().int().nonnegative(), + incomplete: z.number().int().nonnegative(), + eligible: z + .number() + .int() + .nonnegative() + .describe( + 'Tracked owners meeting parity soak timing (matching_since age + fresh checked_at, zero mismatch/error). Does not require the read-cutover feature flag.', + ), + oldestMatchingSince: z.string().nullable(), + newestMatchingSince: z.string().nullable(), + oldestCheckedAt: z.string().nullable(), + newestCheckedAt: z.string().nullable(), + earliestCutoverAt: z + .string() + .nullable() + .describe( + 'Earliest matching_since + soak duration among matching owners, or null.', + ), +}) + +const reconcileMetricsSchema = z.object({ + scanned: z.number().int().nonnegative(), + backfilled: z.number().int().nonnegative(), + compared: z.number().int().nonnegative(), + matched: z.number().int().nonnegative(), + mismatched: z.number().int().nonnegative(), + failed: z.number().int().nonnegative(), +}) + +const retentionMetricsSchema = z.object({ + ownersAttempted: z.number().int().nonnegative(), + ownersSucceeded: z.number().int().nonnegative(), + ownersFailed: z.number().int().nonnegative(), + messagesDeleted: z.number().int().nonnegative(), + threadsDeleted: z.number().int().nonnegative(), + attachmentsDeleted: z.number().int().nonnegative(), + deliveryEventsDeleted: z.number().int().nonnegative(), + blobDeleteFailureOwners: z.number().int().nonnegative(), + expiredRemainingOwners: z.number().int().nonnegative(), +}) + +const inputSchema = z.discriminatedUnion('action', [ + z + .object({ + action: z.literal('status'), + }) + .strict(), + z + .object({ + action: z.literal('reconcile'), + batch_size: batchSizeSchema, + }) + .strict(), + z + .object({ + action: z.literal('retention'), + batch_size: batchSizeSchema, + }) + .strict(), +]) + +const outputSchema = z.discriminatedUnion('action', [ + z.object({ + action: z.literal('status'), + status: statusSchema, + }), + z.object({ + action: z.literal('reconcile'), + metrics: reconcileMetricsSchema, + status: statusSchema, + }), + z.object({ + action: z.literal('retention'), + metrics: retentionMetricsSchema, + status: statusSchema, + }), +]) + +export const adminMailboxMaintenanceCapability = defineDomainCapability( + capabilityDomainNames.admin, + { + ...adminMutationCapabilityAccess, + name: 'admin_mailbox_maintenance', + description: + 'Admin-only Mailbox parity/retention maintenance: aggregate status (no email content), bounded reconcileMailboxParity, or bounded owner-discovered natural retention passes via Mailbox.runRetentionNow. Never accepts arbitrary cutoffs or seed data. Audited.', + keywords: [ + 'admin', + 'mailbox', + 'maintenance', + 'parity', + 'reconcile', + 'retention', + 'cutover', + 'soak', + ], + destructive: true, + inputSchema, + outputSchema, + async handler(args, ctx) { + return auditAdminCapabilityInvocation( + ctx, + 'admin_mailbox_maintenance', + async () => { + switch (args.action) { + case 'status': { + const status = await loadAdminMailboxMaintenanceStatus({ + db: ctx.env.APP_DB, + }) + return { action: 'status' as const, status } + } + case 'reconcile': { + const result = await runAdminMailboxMaintenanceReconcile({ + env: ctx.env, + batchSize: args.batch_size, + }) + return { + action: 'reconcile' as const, + metrics: result.metrics, + status: result.status, + } + } + case 'retention': { + const result = await runAdminMailboxMaintenanceRetention({ + env: ctx.env, + batchSize: args.batch_size, + }) + return { + action: 'retention' as const, + metrics: result.metrics, + status: result.status, + } + } + default: { + const exhaustive: never = args + throw new Error( + `Unhandled admin_mailbox_maintenance action: ${JSON.stringify(exhaustive)}`, + ) + } + } + }, + { + successReason: (result) => { + switch (result.action) { + case 'status': + return `action=status;tracked=${result.status.trackedOwners}` + case 'reconcile': + return `action=reconcile;scanned=${result.metrics.scanned};matched=${result.metrics.matched}` + case 'retention': + return `action=retention;attempted=${result.metrics.ownersAttempted};succeeded=${result.metrics.ownersSucceeded}` + default: { + const exhaustive: never = result + return `action=unknown;${JSON.stringify(exhaustive)}` + } + } + }, + }, + ) + }, + }, +) diff --git a/packages/worker/src/mcp/capabilities/admin/domain.ts b/packages/worker/src/mcp/capabilities/admin/domain.ts index f562e7d0a5..f6d6eed510 100644 --- a/packages/worker/src/mcp/capabilities/admin/domain.ts +++ b/packages/worker/src/mcp/capabilities/admin/domain.ts @@ -29,11 +29,12 @@ import { adminUserUpdateCapability } from './admin-user-update.ts' import { adminAccountWriteLeaseListCapability } from './admin-account-write-lease-list.ts' import { adminAccountWriteLeaseRepairCapability } from './admin-account-write-lease-repair.ts' import { adminUserMeterParityCapability } from './admin-user-meter-parity.ts' +import { adminMailboxMaintenanceCapability } from './admin-mailbox-maintenance.ts' export const adminDomain = defineDomain({ name: capabilityDomainNames.admin, description: - 'Admin-only operator capabilities for account metadata, platform accounts, package scope grants, fleet package-codemod scan/dry-run/apply/revert over published package trees, feature flags, operator-owned system email, attributed platform feedback users explicitly submit for admin review, UserMeter↔D1 parity verification, and metadata about activity on public community listings; never exposes private package source or unrelated user content such as secrets, memories, jobs, or user inbox email.', + 'Admin-only operator capabilities for account metadata, platform accounts, package scope grants, fleet package-codemod scan/dry-run/apply/revert over published package trees, feature flags, operator-owned system email, attributed platform feedback users explicitly submit for admin review, UserMeter↔D1 parity verification, Mailbox parity/retention maintenance aggregates (no inbox content), and metadata about activity on public community listings; never exposes private package source or unrelated user content such as secrets, memories, jobs, or user inbox email.', keywords: [ 'admin', 'rbac', @@ -55,6 +56,9 @@ export const adminDomain = defineDomain({ 'user meter', 'parity', 'cutover', + 'mailbox', + 'maintenance', + 'retention', ], capabilities: [ adminUserListCapability, @@ -62,6 +66,7 @@ export const adminDomain = defineDomain({ adminUserCreateCapability, adminUserUpdateCapability, adminUserMeterParityCapability, + adminMailboxMaintenanceCapability, adminAccountWriteLeaseListCapability, adminAccountWriteLeaseRepairCapability, adminPlatformAccountCreateCapability, diff --git a/packages/worker/src/mcp/capabilities/registry.node.test.ts b/packages/worker/src/mcp/capabilities/registry.node.test.ts index ab53ac3ba4..59a0400566 100644 --- a/packages/worker/src/mcp/capabilities/registry.node.test.ts +++ b/packages/worker/src/mcp/capabilities/registry.node.test.ts @@ -111,11 +111,15 @@ test('getCapabilityRegistryForContext filters admin capabilities by current call expect(adminRegistry.capabilityMap.admin_user_list).toBeTruthy() expect(adminRegistry.capabilityMap.admin_user_meter_parity).toBeTruthy() + expect(adminRegistry.capabilityMap.admin_mailbox_maintenance).toBeTruthy() expect( adminRegistry.capabilityDomains.some((domain) => domain.name === 'admin'), ).toBe(true) expect(regularRegistry.capabilityMap.admin_user_list).toBeUndefined() expect(regularRegistry.capabilityMap.admin_user_meter_parity).toBeUndefined() + expect( + regularRegistry.capabilityMap.admin_mailbox_maintenance, + ).toBeUndefined() expect( regularRegistry.capabilityDomains.some((domain) => domain.name === 'admin'), ).toBe(false) From ab5aaf84dfed5472fe4ebb9887a6791482ccd76e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 20:31:19 +0000 Subject: [PATCH 35/62] fix(admin): bound Mailbox retention maintenance Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 18 +- .../admin/mailbox-maintenance.node.test.ts | 318 ++++++++++++++--- .../worker/src/admin/mailbox-maintenance.ts | 337 +++++++++++++++--- .../admin-mailbox-maintenance.node.test.ts | 98 +++-- .../admin/admin-mailbox-maintenance.ts | 76 +++- 5 files changed, 687 insertions(+), 160 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 7f465324fe..e93b1144e8 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -958,10 +958,20 @@ content). discriminated actions — `status` (aggregate tracked/matching/mismatch/error/ incomplete/eligible counts plus matching/check timestamps and earliest cutover; no email content), `reconcile` (bounded `reconcileMailboxParity`, `batch_size` -max 100, then status), and `retention` (D1-discover bounded non-deleting -non-system owners with mail/parity state, call `runRetentionNow` per owner, -aggregate deletes/failures without owner or message ids). No seed or -arbitrary-cutoff surface. +max 100, then status), and `retention` (natural cutoffs only): + +1. Run existing D1 `pruneUserEmailMessagesForRetention` then + `pruneEmailDeliveryEventsForRetention` (bounded batches; message prune keeps + blob-before-row authority during expand). +2. Keyset-page non-deleting non-system owners with mail/parity state by + `stable_user_id ASC` (`start_after_user_id` / `nextStartAfter` / `truncated`; + never parity `checked_at` ordering; `limit` default/max 20). +3. Call owner-bound `Mailbox.runRetentionNow` with concurrency ≤4 and a ~10s + wall budget (stop scheduling new owners after the deadline; cursor is the + last attempted owner). Per-owner failures are isolated. + +Returns D1 delete/error totals plus aggregate Mailbox before/after counts (no +message ids or email content). No seed or arbitrary-cutoff surface. Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts index 6742eff443..cb9b9fe7e7 100644 --- a/packages/worker/src/admin/mailbox-maintenance.node.test.ts +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -8,12 +8,15 @@ import { mailboxReadCutoverCheckedAtMaxAgeMs, mailboxReadCutoverSoakMs, } from '#worker/email/mailbox-read-cutover.ts' +import type * as RetentionModule from '#app/retention.ts' import type * as MailboxClientModule from '#worker/email/mailbox-client.ts' import type * as MailboxReconcileModule from '#worker/email/mailbox-reconcile.ts' const mocks = vi.hoisted(() => ({ reconcileMailboxParity: vi.fn(), mailboxRpc: vi.fn(), + pruneUserEmailMessagesForRetention: vi.fn(), + pruneEmailDeliveryEventsForRetention: vi.fn(), })) vi.mock('#worker/email/mailbox-reconcile.ts', async (importOriginal) => { @@ -32,8 +35,22 @@ vi.mock('#worker/email/mailbox-client.ts', async (importOriginal) => { } }) +vi.mock('#app/retention.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + pruneUserEmailMessagesForRetention: + mocks.pruneUserEmailMessagesForRetention, + pruneEmailDeliveryEventsForRetention: + mocks.pruneEmailDeliveryEventsForRetention, + } +}) + const { adminMailboxMaintenanceMaxBatchSize, + adminMailboxMaintenanceRetentionConcurrency, + adminMailboxMaintenanceRetentionMaxLimit, + listUsersForAdminMailboxRetention, loadAdminMailboxMaintenanceStatus, runAdminMailboxMaintenanceReconcile, runAdminMailboxMaintenanceRetention, @@ -120,6 +137,31 @@ function insertTrackedUser( return stableUserId } +function stubD1Prune() { + mocks.pruneUserEmailMessagesForRetention.mockResolvedValue({ + deletedMessages: 2, + deletedAttachments: 1, + deletedThreads: 1, + deletedRawMimeBlobs: 2, + deletedAttachmentBlobs: 1, + blobDeleteErrors: 0, + hasMore: false, + nextCursor: null, + }) + mocks.pruneEmailDeliveryEventsForRetention.mockResolvedValue({ + selected: 3, + deleted: 3, + }) +} + +function envFor(db: D1Database) { + return { + APP_DB: db, + MAILBOX: {}, + EMAIL_BLOBS: { delete: vi.fn(async () => undefined) }, + } as unknown as Env +} + test('loadAdminMailboxMaintenanceStatus aggregates buckets without owner ids', async () => { const { sqlite, db } = createMaintenanceDb() const matchingSince = new Date( @@ -178,16 +220,7 @@ test('loadAdminMailboxMaintenanceStatus aggregates buckets without owner ids', a error: 1, incomplete: 1, eligible: 1, - oldestMatchingSince: matchingSince, - newestMatchingSince: matchingSince, - earliestCutoverAt: new Date( - Date.parse(matchingSince) + mailboxReadCutoverSoakMs, - ).toISOString(), }) - const serialized = JSON.stringify(status) - expect(serialized).not.toContain('matching@example.com') - expect(serialized).not.toContain(systemEmailOwnerId) - expect(serialized).not.toContain('countMailbox timed out') }) test('runAdminMailboxMaintenanceReconcile clamps batch_size and returns status', async () => { @@ -200,89 +233,262 @@ test('runAdminMailboxMaintenanceReconcile clamps batch_size and returns status', mismatched: 0, failed: 0, }) - const env = { APP_DB: db, MAILBOX: {} } as unknown as Env const result = await runAdminMailboxMaintenanceReconcile({ - env, + env: envFor(db), batchSize: 10_000, now, }) expect(mocks.reconcileMailboxParity).toHaveBeenCalledWith({ - env, + env: expect.anything(), now, batchSize: adminMailboxMaintenanceMaxBatchSize, }) expect(result.metrics.matched).toBe(1) - expect(result.status.trackedOwners).toBe(0) }) -test('runAdminMailboxMaintenanceRetention aggregates owner results without ids', async () => { - consoleWarn.mockImplementation(() => {}) +test('retention discovery is stable_user_id keyset ordered, not parity checked_at', async () => { const { sqlite, db } = createMaintenanceDb() - const ownerA = insertTrackedUser(sqlite, { - email: 'ret-a@example.com', - checkedAt: '2026-07-01T00:00:00.000Z', + // Lexicographically: aaa... < fff... but checked_at would reverse them. + const laterChecked = insertTrackedUser(sqlite, { + email: 'a-owner@example.com', + stableUserId: 'a'.repeat(64), + checkedAt: '2026-07-20T00:00:00.000Z', }) - const ownerB = insertTrackedUser(sqlite, { - email: 'ret-b@example.com', - checkedAt: '2026-07-02T00:00:00.000Z', + const earlierChecked = insertTrackedUser(sqlite, { + email: 'f-owner@example.com', + stableUserId: 'f'.repeat(64), + checkedAt: '2026-07-01T00:00:00.000Z', }) sqlite .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) - .run('m1', ownerA) + .run('m1', laterChecked) sqlite .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) - .run('m2', ownerB) + .run('m2', earlierChecked) - const runRetentionNow = vi - .fn() - .mockResolvedValueOnce({ + const firstPage = await listUsersForAdminMailboxRetention({ + db, + limit: 1, + }) + expect(firstPage.map((row) => row.userId)).toEqual([laterChecked]) + const secondPage = await listUsersForAdminMailboxRetention({ + db, + limit: 1, + startAfterUserId: laterChecked, + }) + expect(secondPage.map((row) => row.userId)).toEqual([earlierChecked]) + const afterLast = await listUsersForAdminMailboxRetention({ + db, + limit: 1, + startAfterUserId: earlierChecked, + }) + expect(afterLast).toEqual([]) +}) + +test('retention runs D1 prune before DO, pages by cursor, isolates failures', async () => { + consoleWarn.mockImplementation(() => {}) + const { sqlite, db } = createMaintenanceDb() + const callOrder: Array = [] + const ownerA = 'a'.repeat(64) + const ownerB = 'b'.repeat(64) + const ownerC = 'c'.repeat(64) + for (const [id, email] of [ + [ownerA, 'a@example.com'], + [ownerB, 'b@example.com'], + [ownerC, 'c@example.com'], + ] as const) { + insertTrackedUser(sqlite, { + email, + stableUserId: id, + checkedAt: '2026-07-01T00:00:00.000Z', + }) + sqlite + .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) + .run(`m-${id.slice(0, 4)}`, id) + } + + mocks.pruneUserEmailMessagesForRetention.mockImplementation(async () => { + callOrder.push('d1-messages') + return { + deletedMessages: 2, + deletedAttachments: 1, + deletedThreads: 0, + deletedRawMimeBlobs: 2, + deletedAttachmentBlobs: 1, + blobDeleteErrors: 1, + hasMore: false, + nextCursor: null, + } + }) + mocks.pruneEmailDeliveryEventsForRetention.mockImplementation(async () => { + callOrder.push('d1-events') + return { selected: 4, deleted: 4 } + }) + + const runRetentionNow = vi.fn(async ({ ownerId }: { ownerId: string }) => { + callOrder.push(`do:${ownerId.slice(0, 1)}`) + if (ownerId === ownerB) throw new Error('mailbox unavailable') + return { before: { threads: 1, messages: 2, attachments: 1, - deliveryEvents: 3, + deliveryEvents: 1, }, after: { threads: 0, messages: 1, attachments: 0, - deliveryEvents: 1, + deliveryEvents: 0, }, - blobDeleteFailures: true, - expiredRemaining: true, + blobDeleteFailures: false, + expiredRemaining: false, + } + }) + mocks.mailboxRpc.mockImplementation(({ userId }: { userId: string }) => { + callOrder.push(`rpc:${userId.slice(0, 1)}`) + return { runRetentionNow: () => runRetentionNow({ ownerId: userId }) } + }) + + const first = await runAdminMailboxMaintenanceRetention({ + env: envFor(db), + limit: 2, + now, + }) + expect(callOrder.slice(0, 2)).toEqual(['d1-messages', 'd1-events']) + expect(callOrder.indexOf('d1-messages')).toBeLessThan( + callOrder.findIndex((entry) => entry.startsWith('rpc:')), + ) + expect(first.truncated).toBe(true) + expect(first.nextStartAfter).toBe(ownerB) + expect(first.metrics.d1).toEqual({ + messagesDeleted: 2, + attachmentsDeleted: 1, + threadsDeleted: 0, + deliveryEventsDeleted: 4, + rawMimeBlobsDeleted: 2, + attachmentBlobsDeleted: 1, + blobDeleteErrors: 1, + }) + expect(first.metrics.mailbox.ownersAttempted).toBe(2) + expect(first.metrics.mailbox.ownersSucceeded).toBe(1) + expect(first.metrics.mailbox.ownersFailed).toBe(1) + expect(first.metrics.mailbox.before).toEqual({ + threads: 1, + messages: 2, + attachments: 1, + deliveryEvents: 1, + }) + expect(first.metrics.mailbox.after).toEqual({ + threads: 0, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }) + expect(JSON.stringify(first)).not.toContain('mailbox unavailable') + + callOrder.length = 0 + stubD1Prune() + mocks.pruneUserEmailMessagesForRetention.mockImplementation(async () => { + callOrder.push('d1-messages') + return { + deletedMessages: 0, + deletedAttachments: 0, + deletedThreads: 0, + deletedRawMimeBlobs: 0, + deletedAttachmentBlobs: 0, + blobDeleteErrors: 0, + hasMore: false, + nextCursor: null, + } + }) + mocks.pruneEmailDeliveryEventsForRetention.mockImplementation(async () => { + callOrder.push('d1-events') + return { selected: 0, deleted: 0 } + }) + + const second = await runAdminMailboxMaintenanceRetention({ + env: envFor(db), + limit: 2, + startAfterUserId: first.nextStartAfter, + now, + }) + expect(second.metrics.mailbox.ownersAttempted).toBe(1) + expect(second.metrics.mailbox.ownersSucceeded).toBe(1) + expect(second.truncated).toBe(false) + expect(second.nextStartAfter).toBeNull() + expect(runRetentionNow.mock.calls.map((call) => call[0]?.ownerId)).toEqual([ + ownerA, + ownerB, + ownerC, + ]) +}) + +test('retention concurrency stays at most 4 and budget truncates with cursor', async () => { + consoleWarn.mockImplementation(() => {}) + stubD1Prune() + const { sqlite, db } = createMaintenanceDb() + const owners = Array.from({ length: 8 }, (_, index) => { + const id = `${index.toString(16).padStart(2, '0')}${'a'.repeat(62)}` + insertTrackedUser(sqlite, { + email: `u${index}@example.com`, + stableUserId: id, + checkedAt: '2026-07-01T00:00:00.000Z', }) - .mockRejectedValueOnce(new Error('mailbox unavailable')) + sqlite + .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) + .run(`m${index}`, id) + return id + }).sort() - mocks.mailboxRpc.mockImplementation(({ userId }: { userId: string }) => { - expect([ownerA, ownerB]).toContain(userId) - return { runRetentionNow } + const emptyCounts = { + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + } + let inFlight = 0 + let maxInFlight = 0 + let scheduled = 0 + mocks.mailboxRpc.mockImplementation(() => { + scheduled += 1 + return { + runRetentionNow: async () => { + inFlight += 1 + maxInFlight = Math.max(maxInFlight, inFlight) + await Promise.resolve() + inFlight -= 1 + return { + before: emptyCounts, + after: emptyCounts, + blobDeleteFailures: false, + expiredRemaining: false, + } + }, + } }) const result = await runAdminMailboxMaintenanceRetention({ - env: { APP_DB: db, MAILBOX: {} } as unknown as Env, - batchSize: 10, + env: envFor(db), + limit: adminMailboxMaintenanceRetentionMaxLimit, now, + budgetMs: 100, + // After the first concurrency window is scheduled, expire the budget so + // no further owners are launched. + nowMs: () => + scheduled >= adminMailboxMaintenanceRetentionConcurrency ? 1_000 : 0, }) - expect(runRetentionNow).toHaveBeenCalledTimes(2) - expect(runRetentionNow).toHaveBeenCalledWith({ ownerId: ownerA }) - expect(result.metrics).toEqual({ - ownersAttempted: 2, - ownersSucceeded: 1, - ownersFailed: 1, - messagesDeleted: 1, - threadsDeleted: 1, - attachmentsDeleted: 1, - deliveryEventsDeleted: 2, - blobDeleteFailureOwners: 1, - expiredRemainingOwners: 1, - }) - const serialized = JSON.stringify(result) - expect(serialized).not.toContain(ownerA) - expect(serialized).not.toContain(ownerB) - expect(serialized).not.toContain('mailbox unavailable') - expect(consoleWarn).toHaveBeenCalledWith( - 'admin-mailbox-maintenance-retention-owner-failed', - expect.objectContaining({ error: expect.any(Error) }), + expect(maxInFlight).toBeLessThanOrEqual( + adminMailboxMaintenanceRetentionConcurrency, + ) + expect(maxInFlight).toBe(adminMailboxMaintenanceRetentionConcurrency) + expect(result.metrics.mailbox.ownersAttempted).toBe( + adminMailboxMaintenanceRetentionConcurrency, + ) + expect(result.truncated).toBe(true) + expect(result.nextStartAfter).toBe( + owners[adminMailboxMaintenanceRetentionConcurrency - 1], ) + expect(result.metrics.mailbox.ownersAttempted).toBeLessThan(owners.length) }) diff --git a/packages/worker/src/admin/mailbox-maintenance.ts b/packages/worker/src/admin/mailbox-maintenance.ts index e180d01d43..dda83b862b 100644 --- a/packages/worker/src/admin/mailbox-maintenance.ts +++ b/packages/worker/src/admin/mailbox-maintenance.ts @@ -1,3 +1,7 @@ +import { + pruneEmailDeliveryEventsForRetention, + pruneUserEmailMessagesForRetention, +} from '#app/retention.ts' import { systemEmailOwnerId } from '#worker/email/email-owner.ts' import { mailboxRpc, type MailboxEnv } from '#worker/email/mailbox-client.ts' import { @@ -6,16 +10,30 @@ import { type MailboxParityReconcileEnv, type MailboxParityReconcileMetrics, } from '#worker/email/mailbox-reconcile.ts' -import { listUsersForMailboxParity } from '#worker/email/mailbox-parity-repo.ts' import { mailboxReadCutoverCheckedAtMaxAgeMs, mailboxReadCutoverSoakMs, } from '#worker/email/mailbox-read-cutover.ts' +import { type MailboxCountResult } from '#worker/email/mailbox-types.ts' -/** Cap for admin reconcile/retention batch discovery (hard max). */ +/** Cap for admin reconcile batch discovery (hard max). */ export const adminMailboxMaintenanceMaxBatchSize = 100 -export type AdminMailboxMaintenanceEnv = MailboxParityReconcileEnv & MailboxEnv +/** Retention owner page size default/max (hard max). */ +export const adminMailboxMaintenanceRetentionMaxLimit = 20 +export const adminMailboxMaintenanceRetentionDefaultLimit = + adminMailboxMaintenanceRetentionMaxLimit + +/** Max concurrent per-owner Mailbox retention RPCs. */ +export const adminMailboxMaintenanceRetentionConcurrency = 4 + +/** Wall-clock budget for one retention action call. */ +export const adminMailboxMaintenanceRetentionBudgetMs = 10_000 + +export type AdminMailboxMaintenanceEnv = MailboxParityReconcileEnv & + MailboxEnv & { + EMAIL_BLOBS: Pick + } export type AdminMailboxMaintenanceStatus = { generatedAt: string @@ -33,18 +51,31 @@ export type AdminMailboxMaintenanceStatus = { earliestCutoverAt: string | null } -export type AdminMailboxMaintenanceRetentionMetrics = { - ownersAttempted: number - ownersSucceeded: number - ownersFailed: number +export type AdminMailboxMaintenanceRetentionD1Metrics = { messagesDeleted: number - threadsDeleted: number attachmentsDeleted: number + threadsDeleted: number deliveryEventsDeleted: number + rawMimeBlobsDeleted: number + attachmentBlobsDeleted: number + blobDeleteErrors: number +} + +export type AdminMailboxMaintenanceRetentionMailboxMetrics = { + ownersAttempted: number + ownersSucceeded: number + ownersFailed: number + before: MailboxCountResult + after: MailboxCountResult blobDeleteFailureOwners: number expiredRemainingOwners: number } +export type AdminMailboxMaintenanceRetentionMetrics = { + d1: AdminMailboxMaintenanceRetentionD1Metrics + mailbox: AdminMailboxMaintenanceRetentionMailboxMetrics +} + const trackedOwnerSql = ` u.deleting_at IS NULL AND u.stable_user_id != ? @@ -71,7 +102,7 @@ const trackedOwnerSql = ` ) ` -function clampBatchSize(batchSize: number | undefined): number { +function clampReconcileBatchSize(batchSize: number | undefined): number { const requested = batchSize ?? mailboxParityUserBatchSize if (!Number.isFinite(requested)) return mailboxParityUserBatchSize return Math.max( @@ -80,13 +111,143 @@ function clampBatchSize(batchSize: number | undefined): number { ) } -function countDelta(before: number, after: number): number { - return Math.max(0, before - after) +function clampRetentionLimit(limit: number | undefined): number { + const requested = limit ?? adminMailboxMaintenanceRetentionDefaultLimit + if (!Number.isFinite(requested)) { + return adminMailboxMaintenanceRetentionDefaultLimit + } + return Math.max( + 1, + Math.min(adminMailboxMaintenanceRetentionMaxLimit, Math.trunc(requested)), + ) +} + +function emptyMailboxCounts(): MailboxCountResult { + return { threads: 0, messages: 0, attachments: 0, deliveryEvents: 0 } +} + +function addMailboxCounts( + left: MailboxCountResult, + right: MailboxCountResult, +): MailboxCountResult { + return { + threads: left.threads + right.threads, + messages: left.messages + right.messages, + attachments: left.attachments + right.attachments, + deliveryEvents: left.deliveryEvents + right.deliveryEvents, + } +} + +/** + * List non-deleting owners with mail/parity state ordered by stable_user_id + * keyset (never by parity checked_at). + */ +export async function listUsersForAdminMailboxRetention(input: { + db: D1Database + limit: number + startAfterUserId?: string | null +}): Promise> { + const startAfter = input.startAfterUserId ?? null + const result = startAfter + ? await input.db + .prepare( + `SELECT u.stable_user_id AS userId + FROM users u + WHERE ${trackedOwnerSql} + AND u.stable_user_id > ? + ORDER BY u.stable_user_id ASC + LIMIT ?`, + ) + .bind(systemEmailOwnerId, startAfter, input.limit) + .all<{ userId: string }>() + : await input.db + .prepare( + `SELECT u.stable_user_id AS userId + FROM users u + WHERE ${trackedOwnerSql} + ORDER BY u.stable_user_id ASC + LIMIT ?`, + ) + .bind(systemEmailOwnerId, input.limit) + .all<{ userId: string }>() + return result.results ?? [] +} + +type OwnerRetentionOutcome = + | { + status: 'succeeded' + before: MailboxCountResult + after: MailboxCountResult + blobDeleteFailures: boolean + expiredRemaining: boolean + } + | { status: 'failed' } + +async function runOwnersWithBudget(input: { + owners: ReadonlyArray<{ userId: string }> + concurrency: number + deadlineMs: number + nowMs: () => number + runOwner: (userId: string) => Promise +}): Promise<{ + outcomes: Array + lastAttemptedUserId: string | null + stoppedByBudget: boolean +}> { + const outcomes: Array = [] + let nextIndex = 0 + let inFlight = 0 + let lastAttemptedUserId: string | null = null + let stoppedByBudget = false + + await new Promise((resolve) => { + const settleIfDone = () => { + if (inFlight > 0) return + resolve() + } + const launch = () => { + while ( + inFlight < input.concurrency && + nextIndex < input.owners.length && + input.nowMs() < input.deadlineMs + ) { + const owner = input.owners[nextIndex] + if (!owner) break + nextIndex += 1 + lastAttemptedUserId = owner.userId + inFlight += 1 + void input + .runOwner(owner.userId) + .then((outcome) => { + outcomes.push(outcome) + }) + .catch(() => { + // runOwner isolates failures; this is a last-resort guard. + outcomes.push({ status: 'failed' }) + }) + .finally(() => { + inFlight -= 1 + launch() + settleIfDone() + }) + } + if ( + input.nowMs() >= input.deadlineMs && + nextIndex < input.owners.length + ) { + stoppedByBudget = true + } + settleIfDone() + } + launch() + }) + + return { outcomes, lastAttemptedUserId, stoppedByBudget } } /** * Aggregate-only Mailbox parity status for operators. Never returns email - * content, message ids, or per-owner identity. + * content, message ids, or per-owner identity (except retention cursors). */ export async function loadAdminMailboxMaintenanceStatus(input: { db: D1Database @@ -234,7 +395,7 @@ export async function runAdminMailboxMaintenanceReconcile(input: { const metrics = await reconcileMailboxParity({ env: input.env, now, - batchSize: clampBatchSize(input.batchSize), + batchSize: clampReconcileBatchSize(input.batchSize), }) const status = await loadAdminMailboxMaintenanceStatus({ db: input.env.APP_DB, @@ -244,73 +405,133 @@ export async function runAdminMailboxMaintenanceReconcile(input: { } /** - * Bounded D1-discovered retention sweep. Calls owner-bound - * `Mailbox.runRetentionNow` (natural cutoffs only). Aggregates counts without - * owner ids, message ids, or email content. + * Bounded retention sweep: D1 natural-cutoff prune (authoritative blob-before-row + * for messages) then owner-keyed Mailbox DO passes. Stable-id keyset cursor; + * wall budget + concurrency bounds; per-owner failures isolated. */ export async function runAdminMailboxMaintenanceRetention(input: { env: AdminMailboxMaintenanceEnv - batchSize?: number + limit?: number + startAfterUserId?: string | null now?: Date + /** Test seam for wall-clock budget. */ + budgetMs?: number + /** Test seam for concurrency. */ + concurrency?: number + /** Test seam for clock. */ + nowMs?: () => number }): Promise<{ metrics: AdminMailboxMaintenanceRetentionMetrics + nextStartAfter: string | null + truncated: boolean status: AdminMailboxMaintenanceStatus }> { const now = input.now ?? new Date() - const owners = await listUsersForMailboxParity({ + const nowMs = input.nowMs ?? Date.now + const budgetMs = input.budgetMs ?? adminMailboxMaintenanceRetentionBudgetMs + const concurrency = Math.max( + 1, + Math.min( + adminMailboxMaintenanceRetentionConcurrency, + input.concurrency ?? adminMailboxMaintenanceRetentionConcurrency, + ), + ) + const limit = clampRetentionLimit(input.limit) + const startAfterUserId = input.startAfterUserId ?? null + const deadlineMs = nowMs() + budgetMs + + // D1 remains expand-phase authority + blob linkage. Natural cutoffs only. + const d1Messages = await pruneUserEmailMessagesForRetention({ + db: input.env.APP_DB, + blobs: input.env.EMAIL_BLOBS, + now, + }) + const d1Events = await pruneEmailDeliveryEventsForRetention({ + db: input.env.APP_DB, + now, + }) + + const owners = await listUsersForAdminMailboxRetention({ db: input.env.APP_DB, - limit: clampBatchSize(input.batchSize), + limit, + startAfterUserId, }) - const metrics: AdminMailboxMaintenanceRetentionMetrics = { - ownersAttempted: 0, + const { outcomes, lastAttemptedUserId, stoppedByBudget } = + await runOwnersWithBudget({ + owners, + concurrency, + deadlineMs, + nowMs, + async runOwner(userId) { + try { + const result = await mailboxRpc({ + env: input.env, + userId, + }).runRetentionNow({ ownerId: userId }) + return { + status: 'succeeded' as const, + before: result.before, + after: result.after, + blobDeleteFailures: result.blobDeleteFailures, + expiredRemaining: result.expiredRemaining, + } + } catch (error) { + console.warn('admin-mailbox-maintenance-retention-owner-failed', { + error, + }) + return { status: 'failed' as const } + } + }, + }) + + const mailbox: AdminMailboxMaintenanceRetentionMailboxMetrics = { + ownersAttempted: outcomes.length, ownersSucceeded: 0, ownersFailed: 0, - messagesDeleted: 0, - threadsDeleted: 0, - attachmentsDeleted: 0, - deliveryEventsDeleted: 0, + before: emptyMailboxCounts(), + after: emptyMailboxCounts(), blobDeleteFailureOwners: 0, expiredRemainingOwners: 0, } - - for (const { userId } of owners) { - metrics.ownersAttempted += 1 - try { - const result = await mailboxRpc({ - env: input.env, - userId, - }).runRetentionNow({ ownerId: userId }) - metrics.ownersSucceeded += 1 - metrics.messagesDeleted += countDelta( - result.before.messages, - result.after.messages, - ) - metrics.threadsDeleted += countDelta( - result.before.threads, - result.after.threads, - ) - metrics.attachmentsDeleted += countDelta( - result.before.attachments, - result.after.attachments, - ) - metrics.deliveryEventsDeleted += countDelta( - result.before.deliveryEvents, - result.after.deliveryEvents, - ) - if (result.blobDeleteFailures) metrics.blobDeleteFailureOwners += 1 - if (result.expiredRemaining) metrics.expiredRemainingOwners += 1 - } catch (error) { - metrics.ownersFailed += 1 - console.warn('admin-mailbox-maintenance-retention-owner-failed', { - error, - }) + for (const outcome of outcomes) { + if (outcome.status === 'failed') { + mailbox.ownersFailed += 1 + continue } + mailbox.ownersSucceeded += 1 + mailbox.before = addMailboxCounts(mailbox.before, outcome.before) + mailbox.after = addMailboxCounts(mailbox.after, outcome.after) + if (outcome.blobDeleteFailures) mailbox.blobDeleteFailureOwners += 1 + if (outcome.expiredRemaining) mailbox.expiredRemainingOwners += 1 } + const pageFull = owners.length >= limit + const truncated = stoppedByBudget || pageFull + const nextStartAfter = truncated + ? (lastAttemptedUserId ?? startAfterUserId) + : null + const status = await loadAdminMailboxMaintenanceStatus({ db: input.env.APP_DB, now, }) - return { metrics, status } + + return { + metrics: { + d1: { + messagesDeleted: d1Messages.deletedMessages, + attachmentsDeleted: d1Messages.deletedAttachments, + threadsDeleted: d1Messages.deletedThreads, + deliveryEventsDeleted: d1Events.deleted, + rawMimeBlobsDeleted: d1Messages.deletedRawMimeBlobs, + attachmentBlobsDeleted: d1Messages.deletedAttachmentBlobs, + blobDeleteErrors: d1Messages.blobDeleteErrors, + }, + mailbox, + }, + nextStartAfter, + truncated, + status, + } } diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts index c949ddeb26..7c54f5c193 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts @@ -5,6 +5,7 @@ import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.t import { createMcpCallerContext } from '#mcp/context.ts' import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' import type * as MailboxMaintenance from '#worker/admin/mailbox-maintenance.ts' +import { adminMailboxMaintenanceRetentionMaxLimit } from '#worker/admin/mailbox-maintenance.ts' const mockModule = vi.hoisted(() => ({ logAuditEvent: vi.fn(async () => undefined), @@ -54,6 +55,42 @@ const emptyStatus = { earliestCutoverAt: '2026-07-31T00:00:00.000Z', } +const emptyRetentionResult = { + metrics: { + d1: { + messagesDeleted: 1, + attachmentsDeleted: 0, + threadsDeleted: 0, + deliveryEventsDeleted: 2, + rawMimeBlobsDeleted: 1, + attachmentBlobsDeleted: 0, + blobDeleteErrors: 0, + }, + mailbox: { + ownersAttempted: 1, + ownersSucceeded: 1, + ownersFailed: 0, + before: { + threads: 1, + messages: 1, + attachments: 0, + deliveryEvents: 1, + }, + after: { + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }, + blobDeleteFailureOwners: 0, + expiredRemainingOwners: 0, + }, + }, + nextStartAfter: 'a'.repeat(64), + truncated: true, + status: emptyStatus, +} + function createAdminCtx() { const sqlite = new DatabaseSync(':memory:') sqlite.exec(` @@ -124,9 +161,6 @@ test('admin_mailbox_maintenance status is audited and aggregate-only', async () reason: 'action=status;tracked=2', }), ) - const serialized = JSON.stringify(result) - expect(serialized).not.toContain('subject') - expect(serialized).not.toContain('textBody') }) test('admin_mailbox_maintenance reconcile and retention route with schema bounds', async () => { @@ -141,20 +175,9 @@ test('admin_mailbox_maintenance reconcile and retention route with schema bounds }, status: emptyStatus, }) - mockModule.runAdminMailboxMaintenanceRetention.mockResolvedValue({ - metrics: { - ownersAttempted: 1, - ownersSucceeded: 1, - ownersFailed: 0, - messagesDeleted: 0, - threadsDeleted: 0, - attachmentsDeleted: 0, - deliveryEventsDeleted: 0, - blobDeleteFailureOwners: 0, - expiredRemainingOwners: 0, - }, - status: emptyStatus, - }) + mockModule.runAdminMailboxMaintenanceRetention.mockResolvedValue( + emptyRetentionResult, + ) const ctx = createAdminCtx() const reconcile = await adminMailboxMaintenanceCapability.handler( @@ -166,26 +189,29 @@ test('admin_mailbox_maintenance reconcile and retention route with schema bounds env: ctx.env, batchSize: 16, }) - expect(mockModule.logAuditEvent).toHaveBeenCalledWith( - expect.objectContaining({ - action: 'admin_mailbox_maintenance', - reason: 'action=reconcile;scanned=1;matched=1', - }), - ) + const cursor = 'b'.repeat(64) const retention = await adminMailboxMaintenanceCapability.handler( - { action: 'retention', batch_size: 8 }, + { + action: 'retention', + limit: 8, + start_after_user_id: cursor, + }, ctx, ) - expect(retention.action).toBe('retention') + expect(retention).toEqual({ + action: 'retention', + ...emptyRetentionResult, + }) expect(mockModule.runAdminMailboxMaintenanceRetention).toHaveBeenCalledWith({ env: ctx.env, - batchSize: 8, + limit: 8, + startAfterUserId: cursor, }) expect(mockModule.logAuditEvent).toHaveBeenCalledWith( expect.objectContaining({ action: 'admin_mailbox_maintenance', - reason: 'action=retention;attempted=1;succeeded=1', + reason: 'action=retention;attempted=1;succeeded=1;truncated=1', }), ) @@ -195,6 +221,15 @@ test('admin_mailbox_maintenance reconcile and retention route with schema bounds ctx, ), ).rejects.toThrow() + await expect( + adminMailboxMaintenanceCapability.handler( + { + action: 'retention', + limit: adminMailboxMaintenanceRetentionMaxLimit + 1, + }, + ctx, + ), + ).rejects.toThrow() await expect( adminMailboxMaintenanceCapability.handler({ action: 'seed' }, ctx), ).rejects.toThrow() @@ -207,4 +242,13 @@ test('admin_mailbox_maintenance reconcile and retention route with schema bounds ctx, ), ).rejects.toThrow() + await expect( + adminMailboxMaintenanceCapability.handler( + { + action: 'retention', + start_after_user_id: 'not-a-stable-id', + }, + ctx, + ), + ).rejects.toThrow() }) diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts index d943b03849..903c2a174d 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts @@ -1,6 +1,7 @@ import { z } from 'zod' import { adminMailboxMaintenanceMaxBatchSize, + adminMailboxMaintenanceRetentionMaxLimit, loadAdminMailboxMaintenanceStatus, runAdminMailboxMaintenanceReconcile, runAdminMailboxMaintenanceRetention, @@ -10,9 +11,10 @@ import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' import { adminMutationCapabilityAccess, auditAdminCapabilityInvocation, + stableUserIdSchema, } from './admin-shared.ts' -const batchSizeSchema = z +const reconcileBatchSizeSchema = z .number() .int() .min(1) @@ -22,6 +24,23 @@ const batchSizeSchema = z `Optional bounded owner batch size (1–${adminMailboxMaintenanceMaxBatchSize}). Defaults to the scheduled parity lane batch size.`, ) +const retentionLimitSchema = z + .number() + .int() + .min(1) + .max(adminMailboxMaintenanceRetentionMaxLimit) + .optional() + .describe( + `Optional owner page size (1–${adminMailboxMaintenanceRetentionMaxLimit}). Defaults to ${adminMailboxMaintenanceRetentionMaxLimit}.`, + ) + +const mailboxCountSchema = z.object({ + threads: z.number().int().nonnegative(), + messages: z.number().int().nonnegative(), + attachments: z.number().int().nonnegative(), + deliveryEvents: z.number().int().nonnegative(), +}) + const statusSchema = z.object({ generatedAt: z.string(), trackedOwners: z.number().int().nonnegative(), @@ -58,15 +77,24 @@ const reconcileMetricsSchema = z.object({ }) const retentionMetricsSchema = z.object({ - ownersAttempted: z.number().int().nonnegative(), - ownersSucceeded: z.number().int().nonnegative(), - ownersFailed: z.number().int().nonnegative(), - messagesDeleted: z.number().int().nonnegative(), - threadsDeleted: z.number().int().nonnegative(), - attachmentsDeleted: z.number().int().nonnegative(), - deliveryEventsDeleted: z.number().int().nonnegative(), - blobDeleteFailureOwners: z.number().int().nonnegative(), - expiredRemainingOwners: z.number().int().nonnegative(), + d1: z.object({ + messagesDeleted: z.number().int().nonnegative(), + attachmentsDeleted: z.number().int().nonnegative(), + threadsDeleted: z.number().int().nonnegative(), + deliveryEventsDeleted: z.number().int().nonnegative(), + rawMimeBlobsDeleted: z.number().int().nonnegative(), + attachmentBlobsDeleted: z.number().int().nonnegative(), + blobDeleteErrors: z.number().int().nonnegative(), + }), + mailbox: z.object({ + ownersAttempted: z.number().int().nonnegative(), + ownersSucceeded: z.number().int().nonnegative(), + ownersFailed: z.number().int().nonnegative(), + before: mailboxCountSchema, + after: mailboxCountSchema, + blobDeleteFailureOwners: z.number().int().nonnegative(), + expiredRemainingOwners: z.number().int().nonnegative(), + }), }) const inputSchema = z.discriminatedUnion('action', [ @@ -78,13 +106,18 @@ const inputSchema = z.discriminatedUnion('action', [ z .object({ action: z.literal('reconcile'), - batch_size: batchSizeSchema, + batch_size: reconcileBatchSizeSchema, }) .strict(), z .object({ action: z.literal('retention'), - batch_size: batchSizeSchema, + limit: retentionLimitSchema, + start_after_user_id: stableUserIdSchema + .optional() + .describe( + 'Stable-user-id keyset cursor from a prior retention nextStartAfter. Ordered by stable_user_id ASC; never parity checked_at.', + ), }) .strict(), ]) @@ -102,6 +135,16 @@ const outputSchema = z.discriminatedUnion('action', [ z.object({ action: z.literal('retention'), metrics: retentionMetricsSchema, + nextStartAfter: stableUserIdSchema + .nullable() + .describe( + 'Pass as start_after_user_id on the next call when truncated is true.', + ), + truncated: z + .boolean() + .describe( + 'True when more owners remain (full page) or the wall-clock budget stopped scheduling.', + ), status: statusSchema, }), ]) @@ -112,7 +155,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( ...adminMutationCapabilityAccess, name: 'admin_mailbox_maintenance', description: - 'Admin-only Mailbox parity/retention maintenance: aggregate status (no email content), bounded reconcileMailboxParity, or bounded owner-discovered natural retention passes via Mailbox.runRetentionNow. Never accepts arbitrary cutoffs or seed data. Audited.', + 'Admin-only Mailbox parity/retention maintenance: aggregate status (no email content), bounded reconcileMailboxParity, or keyset-paged natural retention (D1 authoritative prune then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget). Never accepts arbitrary cutoffs or seed data. Audited.', keywords: [ 'admin', 'mailbox', @@ -152,11 +195,14 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( case 'retention': { const result = await runAdminMailboxMaintenanceRetention({ env: ctx.env, - batchSize: args.batch_size, + limit: args.limit, + startAfterUserId: args.start_after_user_id, }) return { action: 'retention' as const, metrics: result.metrics, + nextStartAfter: result.nextStartAfter, + truncated: result.truncated, status: result.status, } } @@ -176,7 +222,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( case 'reconcile': return `action=reconcile;scanned=${result.metrics.scanned};matched=${result.metrics.matched}` case 'retention': - return `action=retention;attempted=${result.metrics.ownersAttempted};succeeded=${result.metrics.ownersSucceeded}` + return `action=retention;attempted=${result.metrics.mailbox.ownersAttempted};succeeded=${result.metrics.mailbox.ownersSucceeded};truncated=${result.truncated ? 1 : 0}` default: { const exhaustive: never = result return `action=unknown;${JSON.stringify(exhaustive)}` From bb0d8304f923aa5c9696a0e32500349b13179224 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 20:43:26 +0000 Subject: [PATCH 36/62] fix(admin): gate Mailbox retention on owner D1 cleanup Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 13 +- .../admin/mailbox-maintenance.node.test.ts | 122 ++++++++++++++++-- .../worker/src/admin/mailbox-maintenance.ts | 101 +++++++++++++-- .../admin-mailbox-maintenance.node.test.ts | 1 + .../admin/admin-mailbox-maintenance.ts | 9 +- 5 files changed, 217 insertions(+), 29 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e93b1144e8..8fe0f4779e 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -966,9 +966,16 @@ max 100, then status), and `retention` (natural cutoffs only): 2. Keyset-page non-deleting non-system owners with mail/parity state by `stable_user_id ASC` (`start_after_user_id` / `nextStartAfter` / `truncated`; never parity `checked_at` ordering; `limit` default/max 20). -3. Call owner-bound `Mailbox.runRetentionNow` with concurrency ≤4 and a ~10s - wall budget (stop scheduling new owners after the deadline; cursor is the - last attempted owner). Per-owner failures are isolated. +3. Before each owner DO pass, check for remaining natural-cutoff-expired D1 + `email_messages` (`emailMessageRetentionDays` = 365) or + `email_delivery_events` (`emailDeliveryEventRetentionDays` = 90). If any + remain (e.g. omitted by the global oldest-first batch), skip + `runRetentionNow`, count `pendingD1Owners`, and still advance the cursor so + repeated global D1 batches can drain — never DO/R2-delete while D1 expired + rows remain for that owner. +4. Otherwise call owner-bound `Mailbox.runRetentionNow` with concurrency ≤4 and + a ~10s wall budget (stop scheduling new owners after the deadline; cursor is + the last considered owner). Per-owner failures are isolated. Returns D1 delete/error totals plus aggregate Mailbox before/after counts (no message ids or email content). No seed or arbitrary-cutoff surface. diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts index cb9b9fe7e7..521bebb672 100644 --- a/packages/worker/src/admin/mailbox-maintenance.node.test.ts +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -8,6 +8,7 @@ import { mailboxReadCutoverCheckedAtMaxAgeMs, mailboxReadCutoverSoakMs, } from '#worker/email/mailbox-read-cutover.ts' +import { emailMessageRetentionDays } from '#app/retention.ts' import type * as RetentionModule from '#app/retention.ts' import type * as MailboxClientModule from '#worker/email/mailbox-client.ts' import type * as MailboxReconcileModule from '#worker/email/mailbox-reconcile.ts' @@ -47,6 +48,7 @@ vi.mock('#app/retention.ts', async (importOriginal) => { }) const { + adminEmailRetentionCutoffIso, adminMailboxMaintenanceMaxBatchSize, adminMailboxMaintenanceRetentionConcurrency, adminMailboxMaintenanceRetentionMaxLimit, @@ -57,6 +59,7 @@ const { } = await import('./mailbox-maintenance.ts') const now = new Date('2026-08-01T12:00:00.000Z') +const freshCreatedAt = now.toISOString() function createMaintenanceDb() { const sqlite = new DatabaseSync(':memory:') @@ -83,16 +86,29 @@ function createMaintenanceDb() { ); CREATE TABLE email_messages ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL + user_id TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT '${freshCreatedAt}' ); CREATE TABLE email_delivery_events ( id TEXT PRIMARY KEY, - user_id TEXT NOT NULL + user_id TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT '${freshCreatedAt}' ); `) return { sqlite, db: createD1FromSqlite(sqlite) } } +function insertEmailMessage( + sqlite: DatabaseSync, + input: { id: string; userId: string; createdAt?: string }, +) { + sqlite + .prepare( + `INSERT INTO email_messages (id, user_id, created_at) VALUES (?, ?, ?)`, + ) + .run(input.id, input.userId, input.createdAt ?? freshCreatedAt) +} + function insertTrackedUser( sqlite: DatabaseSync, input: { @@ -259,12 +275,8 @@ test('retention discovery is stable_user_id keyset ordered, not parity checked_a stableUserId: 'f'.repeat(64), checkedAt: '2026-07-01T00:00:00.000Z', }) - sqlite - .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) - .run('m1', laterChecked) - sqlite - .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) - .run('m2', earlierChecked) + insertEmailMessage(sqlite, { id: 'm1', userId: laterChecked }) + insertEmailMessage(sqlite, { id: 'm2', userId: earlierChecked }) const firstPage = await listUsersForAdminMailboxRetention({ db, @@ -302,9 +314,7 @@ test('retention runs D1 prune before DO, pages by cursor, isolates failures', as stableUserId: id, checkedAt: '2026-07-01T00:00:00.000Z', }) - sqlite - .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) - .run(`m-${id.slice(0, 4)}`, id) + insertEmailMessage(sqlite, { id: `m-${id.slice(0, 4)}`, userId: id }) } mocks.pruneUserEmailMessagesForRetention.mockImplementation(async () => { @@ -373,6 +383,7 @@ test('retention runs D1 prune before DO, pages by cursor, isolates failures', as expect(first.metrics.mailbox.ownersAttempted).toBe(2) expect(first.metrics.mailbox.ownersSucceeded).toBe(1) expect(first.metrics.mailbox.ownersFailed).toBe(1) + expect(first.metrics.mailbox.pendingD1Owners).toBe(0) expect(first.metrics.mailbox.before).toEqual({ threads: 1, messages: 2, @@ -435,9 +446,7 @@ test('retention concurrency stays at most 4 and budget truncates with cursor', a stableUserId: id, checkedAt: '2026-07-01T00:00:00.000Z', }) - sqlite - .prepare(`INSERT INTO email_messages (id, user_id) VALUES (?, ?)`) - .run(`m${index}`, id) + insertEmailMessage(sqlite, { id: `m${index}`, userId: id }) return id }).sort() @@ -491,4 +500,89 @@ test('retention concurrency stays at most 4 and budget truncates with cursor', a owners[adminMailboxMaintenanceRetentionConcurrency - 1], ) expect(result.metrics.mailbox.ownersAttempted).toBeLessThan(owners.length) + expect(result.metrics.mailbox.pendingD1Owners).toBe(0) +}) + +test('retention skips DO while owner still has expired D1 rows omitted by global batch', async () => { + stubD1Prune() + const { sqlite, db } = createMaintenanceDb() + const pendingOwner = 'a'.repeat(64) + const clearOwner = 'b'.repeat(64) + insertTrackedUser(sqlite, { + email: 'pending@example.com', + stableUserId: pendingOwner, + checkedAt: '2026-07-01T00:00:00.000Z', + }) + insertTrackedUser(sqlite, { + email: 'clear@example.com', + stableUserId: clearOwner, + checkedAt: '2026-07-01T00:00:00.000Z', + }) + const expiredCreatedAt = adminEmailRetentionCutoffIso( + now, + emailMessageRetentionDays + 1, + ) + // Expired row remains after mocked global prune (oldest-batch omitted this owner). + insertEmailMessage(sqlite, { + id: 'expired-pending', + userId: pendingOwner, + createdAt: expiredCreatedAt, + }) + insertEmailMessage(sqlite, { + id: 'fresh-clear', + userId: clearOwner, + createdAt: freshCreatedAt, + }) + + const runRetentionNow = vi.fn(async () => ({ + before: { + threads: 1, + messages: 1, + attachments: 0, + deliveryEvents: 0, + }, + after: { + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }, + blobDeleteFailures: false, + expiredRemaining: false, + })) + mocks.mailboxRpc.mockImplementation(({ userId }: { userId: string }) => ({ + runRetentionNow: () => runRetentionNow({ ownerId: userId }), + })) + + const first = await runAdminMailboxMaintenanceRetention({ + env: envFor(db), + limit: 2, + now, + }) + expect(first.metrics.mailbox.pendingD1Owners).toBe(1) + expect(first.metrics.mailbox.ownersAttempted).toBe(1) + expect(first.metrics.mailbox.ownersSucceeded).toBe(1) + expect(runRetentionNow).toHaveBeenCalledTimes(1) + expect(runRetentionNow).toHaveBeenCalledWith({ ownerId: clearOwner }) + // Cursor still advanced past the pending owner so global D1 batches can drain. + expect(first.nextStartAfter).toBe(clearOwner) + expect(first.truncated).toBe(true) + + // After D1 clears the omitted expired row, DO retention runs for that owner. + sqlite + .prepare(`DELETE FROM email_messages WHERE id = ?`) + .run('expired-pending') + const second = await runAdminMailboxMaintenanceRetention({ + env: envFor(db), + limit: 2, + startAfterUserId: null, + now, + }) + expect(second.metrics.mailbox.pendingD1Owners).toBe(0) + expect(second.metrics.mailbox.ownersSucceeded).toBe(2) + expect(runRetentionNow.mock.calls.map((call) => call[0]?.ownerId)).toEqual([ + clearOwner, + pendingOwner, + clearOwner, + ]) }) diff --git a/packages/worker/src/admin/mailbox-maintenance.ts b/packages/worker/src/admin/mailbox-maintenance.ts index dda83b862b..803ae30af9 100644 --- a/packages/worker/src/admin/mailbox-maintenance.ts +++ b/packages/worker/src/admin/mailbox-maintenance.ts @@ -1,4 +1,6 @@ import { + emailDeliveryEventRetentionDays, + emailMessageRetentionDays, pruneEmailDeliveryEventsForRetention, pruneUserEmailMessagesForRetention, } from '#app/retention.ts' @@ -65,6 +67,11 @@ export type AdminMailboxMaintenanceRetentionMailboxMetrics = { ownersAttempted: number ownersSucceeded: number ownersFailed: number + /** + * Owners skipped because expired D1 message/event rows remain after the + * global prune batches. Cursor still advances; repeated calls drain via D1. + */ + pendingD1Owners: number before: MailboxCountResult after: MailboxCountResult blobDeleteFailureOwners: number @@ -76,6 +83,49 @@ export type AdminMailboxMaintenanceRetentionMetrics = { mailbox: AdminMailboxMaintenanceRetentionMailboxMetrics } +const millisecondsPerDay = 24 * 60 * 60 * 1000 + +/** Same ISO cutoff formula as `#app/retention` `cutoffIso`. */ +export function adminEmailRetentionCutoffIso(now: Date, days: number) { + return new Date(now.getTime() - days * millisecondsPerDay).toISOString() +} + +/** + * True when the owner still has natural-cutoff-expired D1 mail rows. Owners are + * already non-system from discovery — no extra system exclusion here. + */ +export async function ownerHasPendingExpiredD1Retention(input: { + db: D1Database + userId: string + now: Date +}): Promise { + const messageCutoff = adminEmailRetentionCutoffIso( + input.now, + emailMessageRetentionDays, + ) + const eventCutoff = adminEmailRetentionCutoffIso( + input.now, + emailDeliveryEventRetentionDays, + ) + const row = await input.db + .prepare( + `SELECT 1 AS ok + WHERE EXISTS ( + SELECT 1 FROM email_messages + WHERE user_id = ? AND created_at < ? + LIMIT 1 + ) + OR EXISTS ( + SELECT 1 FROM email_delivery_events + WHERE user_id = ? AND created_at < ? + LIMIT 1 + )`, + ) + .bind(input.userId, messageCutoff, input.userId, eventCutoff) + .first<{ ok: number }>() + return row != null +} + const trackedOwnerSql = ` u.deleting_at IS NULL AND u.stable_user_id != ? @@ -182,6 +232,7 @@ type OwnerRetentionOutcome = expiredRemaining: boolean } | { status: 'failed' } + | { status: 'pending_d1' } async function runOwnersWithBudget(input: { owners: ReadonlyArray<{ userId: string }> @@ -406,8 +457,10 @@ export async function runAdminMailboxMaintenanceReconcile(input: { /** * Bounded retention sweep: D1 natural-cutoff prune (authoritative blob-before-row - * for messages) then owner-keyed Mailbox DO passes. Stable-id keyset cursor; - * wall budget + concurrency bounds; per-owner failures isolated. + * for messages), then per-owner expired-D1 checks, then Mailbox DO passes only + * when D1 is clear for that owner. Stable-id keyset cursor; wall budget + + * concurrency bounds; per-owner failures isolated. Pending-D1 owners skip DO + * but still advance the cursor. */ export async function runAdminMailboxMaintenanceRetention(input: { env: AdminMailboxMaintenanceEnv @@ -465,6 +518,17 @@ export async function runAdminMailboxMaintenanceRetention(input: { nowMs, async runOwner(userId) { try { + // Never DO/R2-delete while this owner still has expired D1 rows + // (global oldest-first batches may have omitted them). + if ( + await ownerHasPendingExpiredD1Retention({ + db: input.env.APP_DB, + userId, + now, + }) + ) { + return { status: 'pending_d1' as const } + } const result = await mailboxRpc({ env: input.env, userId, @@ -486,24 +550,39 @@ export async function runAdminMailboxMaintenanceRetention(input: { }) const mailbox: AdminMailboxMaintenanceRetentionMailboxMetrics = { - ownersAttempted: outcomes.length, + ownersAttempted: 0, ownersSucceeded: 0, ownersFailed: 0, + pendingD1Owners: 0, before: emptyMailboxCounts(), after: emptyMailboxCounts(), blobDeleteFailureOwners: 0, expiredRemainingOwners: 0, } for (const outcome of outcomes) { - if (outcome.status === 'failed') { - mailbox.ownersFailed += 1 - continue + switch (outcome.status) { + case 'pending_d1': + mailbox.pendingD1Owners += 1 + break + case 'failed': + mailbox.ownersAttempted += 1 + mailbox.ownersFailed += 1 + break + case 'succeeded': + mailbox.ownersAttempted += 1 + mailbox.ownersSucceeded += 1 + mailbox.before = addMailboxCounts(mailbox.before, outcome.before) + mailbox.after = addMailboxCounts(mailbox.after, outcome.after) + if (outcome.blobDeleteFailures) mailbox.blobDeleteFailureOwners += 1 + if (outcome.expiredRemaining) mailbox.expiredRemainingOwners += 1 + break + default: { + const exhaustive: never = outcome + throw new Error( + `Unhandled retention owner outcome: ${JSON.stringify(exhaustive)}`, + ) + } } - mailbox.ownersSucceeded += 1 - mailbox.before = addMailboxCounts(mailbox.before, outcome.before) - mailbox.after = addMailboxCounts(mailbox.after, outcome.after) - if (outcome.blobDeleteFailures) mailbox.blobDeleteFailureOwners += 1 - if (outcome.expiredRemaining) mailbox.expiredRemainingOwners += 1 } const pageFull = owners.length >= limit diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts index 7c54f5c193..872554c128 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts @@ -70,6 +70,7 @@ const emptyRetentionResult = { ownersAttempted: 1, ownersSucceeded: 1, ownersFailed: 0, + pendingD1Owners: 0, before: { threads: 1, messages: 1, diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts index 903c2a174d..0b7586f313 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts @@ -90,6 +90,13 @@ const retentionMetricsSchema = z.object({ ownersAttempted: z.number().int().nonnegative(), ownersSucceeded: z.number().int().nonnegative(), ownersFailed: z.number().int().nonnegative(), + pendingD1Owners: z + .number() + .int() + .nonnegative() + .describe( + 'Owners skipped because expired D1 email_messages (365d) or email_delivery_events (90d) remain after global prune batches. Cursor still advances.', + ), before: mailboxCountSchema, after: mailboxCountSchema, blobDeleteFailureOwners: z.number().int().nonnegative(), @@ -155,7 +162,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( ...adminMutationCapabilityAccess, name: 'admin_mailbox_maintenance', description: - 'Admin-only Mailbox parity/retention maintenance: aggregate status (no email content), bounded reconcileMailboxParity, or keyset-paged natural retention (D1 authoritative prune then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget). Never accepts arbitrary cutoffs or seed data. Audited.', + 'Admin-only Mailbox parity/retention maintenance: aggregate status (no email content), bounded reconcileMailboxParity, or keyset-paged natural retention (D1 authoritative prune, skip DO while owner still has expired D1 rows, then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget). Never accepts arbitrary cutoffs or seed data. Audited.', keywords: [ 'admin', 'mailbox', From d8cbf7242e2f703d60b7acd7a01b58deed81f552 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 21:23:34 +0000 Subject: [PATCH 37/62] feat(admin): add Mailbox canary delete control Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 15 +- .../admin/mailbox-maintenance.node.test.ts | 214 +++++++++++++++++- .../worker/src/admin/mailbox-maintenance.ts | 101 ++++++++- .../admin-mailbox-maintenance.node.test.ts | 84 +++++++ .../admin/admin-mailbox-maintenance.ts | 68 +++++- 5 files changed, 476 insertions(+), 6 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 8fe0f4779e..071d820b15 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -958,7 +958,8 @@ content). discriminated actions — `status` (aggregate tracked/matching/mismatch/error/ incomplete/eligible counts plus matching/check timestamps and earliest cutover; no email content), `reconcile` (bounded `reconcileMailboxParity`, `batch_size` -max 100, then status), and `retention` (natural cutoffs only): +max 100, then status), `retention` (natural cutoffs only), and `delete_message` +(owner-scoped single-message canary delete): 1. Run existing D1 `pruneUserEmailMessagesForRetention` then `pruneEmailDeliveryEventsForRetention` (bounded batches; message prune keeps @@ -977,8 +978,16 @@ max 100, then status), and `retention` (natural cutoffs only): a ~10s wall budget (stop scheduling new owners after the deadline; cursor is the last considered owner). Per-owner failures are isolated. -Returns D1 delete/error totals plus aggregate Mailbox before/after counts (no -message ids or email content). No seed or arbitrary-cutoff surface. +`delete_message` takes `stable_user_id` + `message_id`, verifies ownership with +`getEmailMessageById` (missing/foreign rejected), collects attachment metadata / +storage keys without reading content, deletes through existing +`deleteEmailMessageById` (`APP_DB` + `EMAIL_BLOBS`), then verifies the D1 +message is absent and every canonical raw-MIME / external-attachment R2 object +is absent via `head`. Returns aggregate booleans/counts only (no addresses, +bodies, filenames, or keys). Audit success reason includes the target ids. + +Retention returns D1 delete/error totals plus aggregate Mailbox before/after +counts (no message ids or email content). No seed or arbitrary-cutoff surface. Account deletion calls `Mailbox.purge()` (one RPC per user, no D1 id scan; result key `mailboxes`). During expand, `purge` clears DO SQLite / alarm state diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts index 521bebb672..928e6df94b 100644 --- a/packages/worker/src/admin/mailbox-maintenance.node.test.ts +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -4,6 +4,11 @@ import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.t import { consoleWarn } from '#worker/test-support/console-spies.ts' import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' import { systemEmailOwnerId } from '#worker/email/email-owner.ts' +import { + emailAttachmentBlobKey, + emailRawMimeKey, + getEmailMessageById, +} from '#worker/email/repo.ts' import { mailboxReadCutoverCheckedAtMaxAgeMs, mailboxReadCutoverSoakMs, @@ -54,6 +59,7 @@ const { adminMailboxMaintenanceRetentionMaxLimit, listUsersForAdminMailboxRetention, loadAdminMailboxMaintenanceStatus, + runAdminMailboxMaintenanceDeleteMessage, runAdminMailboxMaintenanceReconcile, runAdminMailboxMaintenanceRetention, } = await import('./mailbox-maintenance.ts') @@ -174,10 +180,123 @@ function envFor(db: D1Database) { return { APP_DB: db, MAILBOX: {}, - EMAIL_BLOBS: { delete: vi.fn(async () => undefined) }, + EMAIL_BLOBS: { + delete: vi.fn(async () => undefined), + head: vi.fn(async () => null), + }, } as unknown as Env } +function createMemoryEmailBlobs() { + const objects = new Map() + return { + objects, + blobs: { + async put(key: string, value: string | ArrayBuffer | ArrayBufferView) { + const bytes = + typeof value === 'string' + ? new TextEncoder().encode(value) + : value instanceof ArrayBuffer + ? new Uint8Array(value) + : new Uint8Array(value.buffer, value.byteOffset, value.byteLength) + objects.set(key, bytes) + return { key } as R2Object + }, + async delete(key: string | Array) { + for (const entry of Array.isArray(key) ? key : [key]) { + objects.delete(entry) + } + }, + async head(key: string) { + return objects.has(key) ? ({ key } as R2Object) : null + }, + } satisfies Pick, + } +} + +function createDeleteMessageDb() { + const sqlite = new DatabaseSync(':memory:') + sqlite.exec(` + CREATE TABLE email_messages ( + id TEXT PRIMARY KEY, + direction TEXT NOT NULL, + user_id TEXT NOT NULL, + inbox_id TEXT, + thread_id TEXT, + from_address TEXT NOT NULL, + subject TEXT NOT NULL DEFAULT '', + processing_status TEXT NOT NULL, + raw_mime_key TEXT, + text_body TEXT, + html_body TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE email_attachments ( + id TEXT PRIMARY KEY, + message_id TEXT NOT NULL, + filename TEXT, + content_type TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + storage_kind TEXT NOT NULL, + storage_key TEXT, + created_at TEXT NOT NULL + ); + `) + return { sqlite, db: createD1FromSqlite(sqlite) } +} + +async function seedOwnedMessageWithBlobs(input: { + sqlite: DatabaseSync + blobs: Pick + userId: string + messageId: string + attachmentId: string + includeBody?: boolean +}) { + const rawKey = emailRawMimeKey(input.userId, input.messageId) + const attachmentKey = emailAttachmentBlobKey( + input.userId, + input.messageId, + input.attachmentId, + ) + input.sqlite + .prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, processing_status, + raw_mime_key, text_body, html_body, created_at, updated_at + ) VALUES (?, 'outbound', ?, 'sender@example.net', 'Subject', 'sent', ?, ?, ?, ?, ?)`, + ) + .run( + input.messageId, + input.userId, + rawKey, + input.includeBody === false ? null : 'secret body', + null, + freshCreatedAt, + freshCreatedAt, + ) + input.sqlite + .prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, + storage_key, created_at + ) VALUES (?, ?, 'note.txt', 'text/plain', 4, 'external', ?, ?)`, + ) + .run(input.attachmentId, input.messageId, attachmentKey, freshCreatedAt) + input.sqlite + .prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, + storage_key, created_at + ) VALUES (?, ?, 'inline.png', 'image/png', 0, 'raw-mime', NULL, ?)`, + ) + .run(`inline-${input.attachmentId}`, input.messageId, freshCreatedAt) + await input.blobs.put(rawKey, 'raw-mime-bytes') + await input.blobs.put(attachmentKey, 'att') + return { rawKey, attachmentKey } +} + test('loadAdminMailboxMaintenanceStatus aggregates buckets without owner ids', async () => { const { sqlite, db } = createMaintenanceDb() const matchingSince = new Date( @@ -586,3 +705,96 @@ test('retention skips DO while owner still has expired D1 rows omitted by global clearOwner, ]) }) + +test('delete_message enforces owner isolation and verifies D1/R2 linkage aggregates', async () => { + const { sqlite, db } = createDeleteMessageDb() + const { blobs, objects } = createMemoryEmailBlobs() + const ownerA = testStableUserIdFromEmail('owner-a@example.com') + const ownerB = testStableUserIdFromEmail('owner-b@example.com') + const messageA = 'msg-owner-a' + const messageB = 'msg-owner-b' + const attachmentA = 'att-owner-a' + const attachmentB = 'att-owner-b' + const seededA = await seedOwnedMessageWithBlobs({ + sqlite, + blobs, + userId: ownerA, + messageId: messageA, + attachmentId: attachmentA, + }) + const seededB = await seedOwnedMessageWithBlobs({ + sqlite, + blobs, + userId: ownerB, + messageId: messageB, + attachmentId: attachmentB, + }) + const env = { + APP_DB: db, + MAILBOX: {}, + EMAIL_BLOBS: blobs, + } as unknown as Env + + await expect( + runAdminMailboxMaintenanceDeleteMessage({ + env, + stableUserId: ownerA, + messageId: messageB, + }), + ).rejects.toThrow( + `Email message not found for stable_user_id=${ownerA} message_id=${messageB}`, + ) + await expect( + runAdminMailboxMaintenanceDeleteMessage({ + env, + stableUserId: ownerA, + messageId: 'missing-message', + }), + ).rejects.toThrow( + `Email message not found for stable_user_id=${ownerA} message_id=missing-message`, + ) + expect( + await getEmailMessageById({ + db, + userId: ownerB, + messageId: messageB, + }), + ).not.toBeNull() + expect(objects.has(seededB.rawKey)).toBe(true) + expect(objects.has(seededB.attachmentKey)).toBe(true) + + const result = await runAdminMailboxMaintenanceDeleteMessage({ + env, + stableUserId: ownerA, + messageId: messageA, + }) + expect(result).toEqual({ + d1MessageAbsent: true, + attachmentsSeen: 2, + externalAttachmentsSeen: 1, + rawMimeBlobAbsent: true, + externalAttachmentBlobsAbsent: 1, + allCanonicalBlobsAbsent: true, + }) + expect(JSON.stringify(result)).not.toMatch( + /@|secret body|email-raw:|email-attachment:/, + ) + expect( + await getEmailMessageById({ + db, + userId: ownerA, + messageId: messageA, + }), + ).toBeNull() + expect(objects.has(seededA.rawKey)).toBe(false) + expect(objects.has(seededA.attachmentKey)).toBe(false) + expect( + await getEmailMessageById({ + db, + userId: ownerB, + messageId: messageB, + }), + ).not.toBeNull() + expect(objects.has(seededB.rawKey)).toBe(true) + expect(objects.has(seededB.attachmentKey)).toBe(true) +}) diff --git a/packages/worker/src/admin/mailbox-maintenance.ts b/packages/worker/src/admin/mailbox-maintenance.ts index 803ae30af9..d84786af9b 100644 --- a/packages/worker/src/admin/mailbox-maintenance.ts +++ b/packages/worker/src/admin/mailbox-maintenance.ts @@ -17,6 +17,13 @@ import { mailboxReadCutoverSoakMs, } from '#worker/email/mailbox-read-cutover.ts' import { type MailboxCountResult } from '#worker/email/mailbox-types.ts' +import { + deleteEmailMessageById, + emailAttachmentBlobKey, + emailRawMimeKey, + getEmailMessageById, + listEmailAttachmentsForMessage, +} from '#worker/email/repo.ts' /** Cap for admin reconcile batch discovery (hard max). */ export const adminMailboxMaintenanceMaxBatchSize = 100 @@ -34,7 +41,7 @@ export const adminMailboxMaintenanceRetentionBudgetMs = 10_000 export type AdminMailboxMaintenanceEnv = MailboxParityReconcileEnv & MailboxEnv & { - EMAIL_BLOBS: Pick + EMAIL_BLOBS: Pick } export type AdminMailboxMaintenanceStatus = { @@ -83,6 +90,19 @@ export type AdminMailboxMaintenanceRetentionMetrics = { mailbox: AdminMailboxMaintenanceRetentionMailboxMetrics } +/** + * Aggregate-only outcome for a single audited admin message delete. Never + * includes addresses, bodies, filenames, or R2 keys. + */ +export type AdminMailboxMaintenanceDeleteMessageResult = { + d1MessageAbsent: boolean + attachmentsSeen: number + externalAttachmentsSeen: number + rawMimeBlobAbsent: boolean + externalAttachmentBlobsAbsent: number + allCanonicalBlobsAbsent: boolean +} + const millisecondsPerDay = 24 * 60 * 60 * 1000 /** Same ISO cutoff formula as `#app/retention` `cutoffIso`. */ @@ -614,3 +634,82 @@ export async function runAdminMailboxMaintenanceRetention(input: { status, } } + +/** + * Owner-scoped single-message delete for accelerated Mailbox coverage canaries. + * Verifies ownership via getEmailMessageById, collects attachment metadata / + * storage keys (no content), deletes through existing deleteEmailMessageById, + * then confirms D1 absence plus every canonical raw-MIME / external-attachment + * R2 object absent via head. Rejects missing or foreign messages. + */ +export async function runAdminMailboxMaintenanceDeleteMessage(input: { + env: AdminMailboxMaintenanceEnv + stableUserId: string + messageId: string +}): Promise { + const db = input.env.APP_DB + const blobs = input.env.EMAIL_BLOBS + const message = await getEmailMessageById({ + db, + userId: input.stableUserId, + messageId: input.messageId, + }) + if (!message) { + throw new Error( + `Email message not found for stable_user_id=${input.stableUserId} message_id=${input.messageId}`, + ) + } + + // Collect attachment metadata/storage keys before delete (no content). + const attachmentMetadata = ( + await listEmailAttachmentsForMessage({ + db, + messageId: message.id, + }) + ).map((attachment) => ({ + id: attachment.id, + storageKind: attachment.storageKind, + storageKey: attachment.storageKey, + })) + const externalAttachments = attachmentMetadata.filter( + (attachment) => attachment.storageKind === 'external', + ) + const rawMimeKey = emailRawMimeKey(input.stableUserId, message.id) + const canonicalExternalAttachmentKeys = externalAttachments.map( + (attachment) => + emailAttachmentBlobKey(input.stableUserId, message.id, attachment.id), + ) + + await deleteEmailMessageById({ + db, + blobs: blobs as R2Bucket, + messageId: message.id, + }) + + const remaining = await getEmailMessageById({ + db, + userId: input.stableUserId, + messageId: message.id, + }) + const d1MessageAbsent = remaining == null + + const rawMimeBlobAbsent = (await blobs.head(rawMimeKey)) == null + let externalAttachmentBlobsAbsent = 0 + for (const key of canonicalExternalAttachmentKeys) { + if ((await blobs.head(key)) == null) { + externalAttachmentBlobsAbsent += 1 + } + } + const allCanonicalBlobsAbsent = + rawMimeBlobAbsent && + externalAttachmentBlobsAbsent === canonicalExternalAttachmentKeys.length + + return { + d1MessageAbsent, + attachmentsSeen: attachmentMetadata.length, + externalAttachmentsSeen: externalAttachments.length, + rawMimeBlobAbsent, + externalAttachmentBlobsAbsent, + allCanonicalBlobsAbsent, + } +} diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts index 872554c128..6c579d3785 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts @@ -12,6 +12,7 @@ const mockModule = vi.hoisted(() => ({ loadAdminMailboxMaintenanceStatus: vi.fn(), runAdminMailboxMaintenanceReconcile: vi.fn(), runAdminMailboxMaintenanceRetention: vi.fn(), + runAdminMailboxMaintenanceDeleteMessage: vi.fn(), })) vi.mock('#worker/audit-log.ts', async (importOriginal) => { @@ -33,6 +34,8 @@ vi.mock('#worker/admin/mailbox-maintenance.ts', async (importOriginal) => { mockModule.runAdminMailboxMaintenanceReconcile, runAdminMailboxMaintenanceRetention: mockModule.runAdminMailboxMaintenanceRetention, + runAdminMailboxMaintenanceDeleteMessage: + mockModule.runAdminMailboxMaintenanceDeleteMessage, } }) @@ -92,6 +95,15 @@ const emptyRetentionResult = { status: emptyStatus, } +const emptyDeleteResult = { + d1MessageAbsent: true, + attachmentsSeen: 2, + externalAttachmentsSeen: 1, + rawMimeBlobAbsent: true, + externalAttachmentBlobsAbsent: 1, + allCanonicalBlobsAbsent: true, +} + function createAdminCtx() { const sqlite = new DatabaseSync(':memory:') sqlite.exec(` @@ -145,6 +157,9 @@ test('admin_mailbox_maintenance is registered admin-only destructive with keywor expect(adminMailboxMaintenanceCapability.name).toBe( 'admin_mailbox_maintenance', ) + expect(adminMailboxMaintenanceCapability.keywords).toEqual( + expect.arrayContaining(['delete']), + ) }) test('admin_mailbox_maintenance status is audited and aggregate-only', async () => { @@ -253,3 +268,72 @@ test('admin_mailbox_maintenance reconcile and retention route with schema bounds ), ).rejects.toThrow() }) + +test('admin_mailbox_maintenance delete_message is audited with ids and schema-bound', async () => { + mockModule.runAdminMailboxMaintenanceDeleteMessage.mockResolvedValue( + emptyDeleteResult, + ) + const ctx = createAdminCtx() + const stableUserId = testStableUserIdFromEmail('target@example.com') + const messageId = 'canary-message-1' + + const result = await adminMailboxMaintenanceCapability.handler( + { + action: 'delete_message', + stable_user_id: stableUserId, + message_id: messageId, + }, + ctx, + ) + expect(result).toEqual({ + action: 'delete_message', + result: emptyDeleteResult, + }) + expect( + mockModule.runAdminMailboxMaintenanceDeleteMessage, + ).toHaveBeenCalledWith({ + env: ctx.env, + stableUserId, + messageId, + }) + expect(mockModule.logAuditEvent).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'admin_mailbox_maintenance', + result: 'success', + reason: `action=delete_message;stable_user_id=${stableUserId};message_id=${messageId};d1_absent=1;blobs_absent=1`, + }), + ) + expect(JSON.stringify(result)).not.toMatch( + /@example|secret body|email-raw:|email-attachment:/, + ) + + await expect( + adminMailboxMaintenanceCapability.handler( + { + action: 'delete_message', + stable_user_id: 'not-a-stable-id', + message_id: messageId, + }, + ctx, + ), + ).rejects.toThrow() + await expect( + adminMailboxMaintenanceCapability.handler( + { + action: 'delete_message', + stable_user_id: stableUserId, + }, + ctx, + ), + ).rejects.toThrow() + await expect( + adminMailboxMaintenanceCapability.handler( + { + action: 'delete_message', + stable_user_id: stableUserId, + message_id: '', + }, + ctx, + ), + ).rejects.toThrow() +}) diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts index 0b7586f313..925c15d155 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts @@ -3,6 +3,7 @@ import { adminMailboxMaintenanceMaxBatchSize, adminMailboxMaintenanceRetentionMaxLimit, loadAdminMailboxMaintenanceStatus, + runAdminMailboxMaintenanceDeleteMessage, runAdminMailboxMaintenanceReconcile, runAdminMailboxMaintenanceRetention, } from '#worker/admin/mailbox-maintenance.ts' @@ -104,6 +105,37 @@ const retentionMetricsSchema = z.object({ }), }) +const deleteMessageResultSchema = z.object({ + d1MessageAbsent: z + .boolean() + .describe('True when the D1 email_messages row is gone after delete.'), + attachmentsSeen: z + .number() + .int() + .nonnegative() + .describe('Attachment metadata rows observed before delete.'), + externalAttachmentsSeen: z + .number() + .int() + .nonnegative() + .describe('External-storage attachment rows observed before delete.'), + rawMimeBlobAbsent: z + .boolean() + .describe('True when head(emailRawMimeKey) finds no object after delete.'), + externalAttachmentBlobsAbsent: z + .number() + .int() + .nonnegative() + .describe( + 'Count of canonical external attachment keys with no R2 object after delete.', + ), + allCanonicalBlobsAbsent: z + .boolean() + .describe( + 'True when the canonical raw-MIME key and every external attachment key are absent.', + ), +}) + const inputSchema = z.discriminatedUnion('action', [ z .object({ @@ -127,6 +159,18 @@ const inputSchema = z.discriminatedUnion('action', [ ), }) .strict(), + z + .object({ + action: z.literal('delete_message'), + stable_user_id: stableUserIdSchema.describe( + 'Owning users.stable_user_id. Foreign or missing messages are rejected.', + ), + message_id: z + .string() + .min(1) + .describe('email_messages.id to delete for the given owner.'), + }) + .strict(), ]) const outputSchema = z.discriminatedUnion('action', [ @@ -154,6 +198,10 @@ const outputSchema = z.discriminatedUnion('action', [ ), status: statusSchema, }), + z.object({ + action: z.literal('delete_message'), + result: deleteMessageResultSchema, + }), ]) export const adminMailboxMaintenanceCapability = defineDomainCapability( @@ -162,7 +210,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( ...adminMutationCapabilityAccess, name: 'admin_mailbox_maintenance', description: - 'Admin-only Mailbox parity/retention maintenance: aggregate status (no email content), bounded reconcileMailboxParity, or keyset-paged natural retention (D1 authoritative prune, skip DO while owner still has expired D1 rows, then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget). Never accepts arbitrary cutoffs or seed data. Audited.', + 'Admin-only Mailbox parity/retention/delete maintenance: aggregate status (no email content), bounded reconcileMailboxParity, keyset-paged natural retention (D1 authoritative prune, skip DO while owner still has expired D1 rows, then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget), or owner-scoped delete_message (getEmailMessageById ownership check, existing deleteEmailMessageById, post-delete D1/R2 head aggregates). Never accepts arbitrary cutoffs or seed data. Audited.', keywords: [ 'admin', 'mailbox', @@ -170,6 +218,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( 'parity', 'reconcile', 'retention', + 'delete', 'cutover', 'soak', ], @@ -213,6 +262,14 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( status: result.status, } } + case 'delete_message': { + const result = await runAdminMailboxMaintenanceDeleteMessage({ + env: ctx.env, + stableUserId: args.stable_user_id, + messageId: args.message_id, + }) + return { action: 'delete_message' as const, result } + } default: { const exhaustive: never = args throw new Error( @@ -230,6 +287,15 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( return `action=reconcile;scanned=${result.metrics.scanned};matched=${result.metrics.matched}` case 'retention': return `action=retention;attempted=${result.metrics.mailbox.ownersAttempted};succeeded=${result.metrics.mailbox.ownersSucceeded};truncated=${result.truncated ? 1 : 0}` + case 'delete_message': { + const targetStableUserId = + args.action === 'delete_message' + ? args.stable_user_id + : 'unknown' + const targetMessageId = + args.action === 'delete_message' ? args.message_id : 'unknown' + return `action=delete_message;stable_user_id=${targetStableUserId};message_id=${targetMessageId};d1_absent=${result.result.d1MessageAbsent ? 1 : 0};blobs_absent=${result.result.allCanonicalBlobsAbsent ? 1 : 0}` + } default: { const exhaustive: never = result return `action=unknown;${JSON.stringify(exhaustive)}` From aacb7469fd46bba393ef80aaf3e3c1d78b5b5414 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 21:34:48 +0000 Subject: [PATCH 38/62] fix(admin): verify exact Mailbox delete inventory Co-authored-by: Kent C. Dodds --- .../admin/mailbox-maintenance.node.test.ts | 144 +++++++++++++++++- .../worker/src/admin/mailbox-maintenance.ts | 64 ++++---- packages/worker/src/email/repo.ts | 88 +++++++++-- 3 files changed, 247 insertions(+), 49 deletions(-) diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts index 928e6df94b..ad77f025ca 100644 --- a/packages/worker/src/admin/mailbox-maintenance.node.test.ts +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -4,7 +4,9 @@ import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.t import { consoleWarn } from '#worker/test-support/console-spies.ts' import { testStableUserIdFromEmail } from '#worker/test-support/stable-user-id.ts' import { systemEmailOwnerId } from '#worker/email/email-owner.ts' +import * as EmailRepo from '#worker/email/repo.ts' import { + deleteEmailMessageById, emailAttachmentBlobKey, emailRawMimeKey, getEmailMessageById, @@ -774,7 +776,7 @@ test('delete_message enforces owner isolation and verifies D1/R2 linkage aggrega externalAttachmentsSeen: 1, rawMimeBlobAbsent: true, externalAttachmentBlobsAbsent: 1, - allCanonicalBlobsAbsent: true, + allCapturedBlobsAbsent: true, }) expect(JSON.stringify(result)).not.toMatch( /@|secret body|email-raw:|email-attachment:/, @@ -798,3 +800,143 @@ test('delete_message enforces owner isolation and verifies D1/R2 linkage aggrega expect(objects.has(seededB.rawKey)).toBe(true) expect(objects.has(seededB.attachmentKey)).toBe(true) }) + +test('deleteEmailMessageById expectedUserId fence rejects foreign owners without mutating', async () => { + const { sqlite, db } = createDeleteMessageDb() + const { blobs, objects } = createMemoryEmailBlobs() + const owner = testStableUserIdFromEmail('fence-owner@example.com') + const other = testStableUserIdFromEmail('fence-other@example.com') + const messageId = 'msg-fence' + const attachmentId = 'att-fence' + const seeded = await seedOwnedMessageWithBlobs({ + sqlite, + blobs, + userId: owner, + messageId, + attachmentId, + }) + + await expect( + deleteEmailMessageById({ + db, + blobs: blobs as R2Bucket, + messageId, + expectedUserId: other, + }), + ).rejects.toThrow( + `Email message not found for expected_user_id=${other} message_id=${messageId}`, + ) + expect( + await getEmailMessageById({ + db, + userId: owner, + messageId, + }), + ).not.toBeNull() + expect(objects.has(seeded.rawKey)).toBe(true) + expect(objects.has(seeded.attachmentKey)).toBe(true) + + const deletion = await deleteEmailMessageById({ + db, + blobs: blobs as R2Bucket, + messageId, + expectedUserId: owner, + }) + expect(deletion.messageFound).toBe(true) + expect(deletion.ownerUserId).toBe(owner) + expect(deletion.attachmentsSeen).toBe(2) + expect(deletion.externalAttachmentsSeen).toBe(1) + expect(deletion.blobDeletions.map((entry) => entry.key)).toEqual([ + seeded.rawKey, + seeded.attachmentKey, + ]) + expect(deletion.blobDeletions.every((entry) => entry.deleted)).toBe(true) +}) + +test('delete_message head-checks exact keys captured after verification drift', async () => { + const { sqlite, db } = createDeleteMessageDb() + const { blobs, objects } = createMemoryEmailBlobs() + const owner = testStableUserIdFromEmail('drift-owner@example.com') + const messageId = 'msg-drift' + const attachmentId = 'att-original' + const seeded = await seedOwnedMessageWithBlobs({ + sqlite, + blobs, + userId: owner, + messageId, + attachmentId, + }) + const driftAttachmentId = 'att-drifted' + const driftKey = emailAttachmentBlobKey(owner, messageId, driftAttachmentId) + const headedKeys: Array = [] + const trackingBlobs = { + async put(key: string, value: string | ArrayBuffer | ArrayBufferView) { + return blobs.put(key, value) + }, + async delete(key: string | Array) { + return blobs.delete(key) + }, + async head(key: string) { + headedKeys.push(key) + return blobs.head(key) + }, + } + const env = { + APP_DB: db, + MAILBOX: {}, + EMAIL_BLOBS: trackingBlobs, + } as unknown as Env + + const actualGet = EmailRepo.getEmailMessageById + const getSpy = vi + .spyOn(EmailRepo, 'getEmailMessageById') + .mockImplementation(async (input) => { + const message = await actualGet(input) + // Drift after ownership verification returns, before delete capture. + if (message != null && !objects.has(driftKey)) { + sqlite + .prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, + storage_key, created_at + ) VALUES (?, ?, 'drift.txt', 'text/plain', 5, 'external', ?, ?)`, + ) + .run(driftAttachmentId, messageId, driftKey, freshCreatedAt) + await blobs.put(driftKey, 'drift') + } + return message + }) + + try { + const deleteSpy = vi.spyOn(EmailRepo, 'deleteEmailMessageById') + const result = await runAdminMailboxMaintenanceDeleteMessage({ + env, + stableUserId: owner, + messageId, + }) + expect(deleteSpy).toHaveBeenCalledWith( + expect.objectContaining({ + messageId, + expectedUserId: owner, + }), + ) + expect(result).toEqual({ + d1MessageAbsent: true, + attachmentsSeen: 3, + externalAttachmentsSeen: 2, + rawMimeBlobAbsent: true, + externalAttachmentBlobsAbsent: 2, + allCapturedBlobsAbsent: true, + }) + expect(headedKeys).toEqual([seeded.rawKey, seeded.attachmentKey, driftKey]) + expect(objects.has(seeded.rawKey)).toBe(false) + expect(objects.has(seeded.attachmentKey)).toBe(false) + expect(objects.has(driftKey)).toBe(false) + expect(JSON.stringify(result)).not.toMatch( + /@|secret body|email-raw:|email-attachment:/, + ) + deleteSpy.mockRestore() + } finally { + getSpy.mockRestore() + } +}) diff --git a/packages/worker/src/admin/mailbox-maintenance.ts b/packages/worker/src/admin/mailbox-maintenance.ts index d84786af9b..48b5a152b5 100644 --- a/packages/worker/src/admin/mailbox-maintenance.ts +++ b/packages/worker/src/admin/mailbox-maintenance.ts @@ -19,10 +19,7 @@ import { import { type MailboxCountResult } from '#worker/email/mailbox-types.ts' import { deleteEmailMessageById, - emailAttachmentBlobKey, - emailRawMimeKey, getEmailMessageById, - listEmailAttachmentsForMessage, } from '#worker/email/repo.ts' /** Cap for admin reconcile batch discovery (hard max). */ @@ -100,7 +97,8 @@ export type AdminMailboxMaintenanceDeleteMessageResult = { externalAttachmentsSeen: number rawMimeBlobAbsent: boolean externalAttachmentBlobsAbsent: number - allCanonicalBlobsAbsent: boolean + /** True when every blob key captured by deleteEmailMessageById is absent. */ + allCapturedBlobsAbsent: boolean } const millisecondsPerDay = 24 * 60 * 60 * 1000 @@ -637,10 +635,10 @@ export async function runAdminMailboxMaintenanceRetention(input: { /** * Owner-scoped single-message delete for accelerated Mailbox coverage canaries. - * Verifies ownership via getEmailMessageById, collects attachment metadata / - * storage keys (no content), deletes through existing deleteEmailMessageById, - * then confirms D1 absence plus every canonical raw-MIME / external-attachment - * R2 object absent via head. Rejects missing or foreign messages. + * Verifies ownership via getEmailMessageById, deletes through + * deleteEmailMessageById with an expectedUserId fence, then confirms D1 + * absence plus every exact captured blob key absent via head. Rejects missing + * or foreign messages. */ export async function runAdminMailboxMaintenanceDeleteMessage(input: { env: AdminMailboxMaintenanceEnv @@ -660,30 +658,11 @@ export async function runAdminMailboxMaintenanceDeleteMessage(input: { ) } - // Collect attachment metadata/storage keys before delete (no content). - const attachmentMetadata = ( - await listEmailAttachmentsForMessage({ - db, - messageId: message.id, - }) - ).map((attachment) => ({ - id: attachment.id, - storageKind: attachment.storageKind, - storageKey: attachment.storageKey, - })) - const externalAttachments = attachmentMetadata.filter( - (attachment) => attachment.storageKind === 'external', - ) - const rawMimeKey = emailRawMimeKey(input.stableUserId, message.id) - const canonicalExternalAttachmentKeys = externalAttachments.map( - (attachment) => - emailAttachmentBlobKey(input.stableUserId, message.id, attachment.id), - ) - - await deleteEmailMessageById({ + const deletion = await deleteEmailMessageById({ db, blobs: blobs as R2Bucket, messageId: message.id, + expectedUserId: input.stableUserId, }) const remaining = await getEmailMessageById({ @@ -693,23 +672,34 @@ export async function runAdminMailboxMaintenanceDeleteMessage(input: { }) const d1MessageAbsent = remaining == null - const rawMimeBlobAbsent = (await blobs.head(rawMimeKey)) == null + const rawMimeEntries = deletion.blobDeletions.filter( + (entry) => entry.role === 'raw_mime', + ) + const attachmentEntries = deletion.blobDeletions.filter( + (entry) => entry.role === 'attachment', + ) + let rawMimeBlobAbsent = true + for (const entry of rawMimeEntries) { + if ((await blobs.head(entry.key)) != null) { + rawMimeBlobAbsent = false + } + } let externalAttachmentBlobsAbsent = 0 - for (const key of canonicalExternalAttachmentKeys) { - if ((await blobs.head(key)) == null) { + for (const entry of attachmentEntries) { + if ((await blobs.head(entry.key)) == null) { externalAttachmentBlobsAbsent += 1 } } - const allCanonicalBlobsAbsent = + const allCapturedBlobsAbsent = rawMimeBlobAbsent && - externalAttachmentBlobsAbsent === canonicalExternalAttachmentKeys.length + externalAttachmentBlobsAbsent === attachmentEntries.length return { d1MessageAbsent, - attachmentsSeen: attachmentMetadata.length, - externalAttachmentsSeen: externalAttachments.length, + attachmentsSeen: deletion.attachmentsSeen, + externalAttachmentsSeen: deletion.externalAttachmentsSeen, rawMimeBlobAbsent, externalAttachmentBlobsAbsent, - allCanonicalBlobsAbsent, + allCapturedBlobsAbsent, } } diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 0327befafc..754c50504c 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -1060,6 +1060,25 @@ export async function searchEmailMessages(input: { return (result.results ?? []).map(mapMessageRow) } +export type DeleteEmailMessageBlobDeletion = { + key: string + role: 'raw_mime' | 'attachment' + /** True when `blobs.delete` settled without throwing. */ + deleted: boolean +} + +export type DeleteEmailMessageByIdResult = { + messageFound: boolean + ownerUserId: string | null + attachmentsSeen: number + externalAttachmentsSeen: number + /** + * Exact blob keys/outcomes from the R2 delete loop. Keys are for internal + * post-delete verification only and must not be returned to clients. + */ + blobDeletions: Array +} + export async function deleteEmailMessageById(input: { db: D1Database /** @@ -1068,7 +1087,12 @@ export async function deleteEmailMessageById(input: { */ blobs: R2Bucket messageId: string -}) { + /** + * Optional owner fence. When set, refuse to mutate unless the captured + * `email_messages.user_id` matches (missing/foreign → throw). + */ + expectedUserId?: string +}): Promise { // Capture ownership and blob keys before D1 delete. A failed read must // abort the delete (and be retried) rather than orphan the R2 blobs; only // the R2 deletes themselves are best-effort. @@ -1076,16 +1100,44 @@ export async function deleteEmailMessageById(input: { .prepare(`SELECT user_id FROM email_messages WHERE id = ?`) .bind(input.messageId) .first<{ user_id: string }>() + if ( + input.expectedUserId !== undefined && + (row == null || row.user_id !== input.expectedUserId) + ) { + throw new Error( + `Email message not found for expected_user_id=${input.expectedUserId} message_id=${input.messageId}`, + ) + } const attachmentRows = await input.db .prepare( - `SELECT storage_key FROM email_attachments - WHERE message_id = ? AND storage_key IS NOT NULL`, + `SELECT storage_kind, storage_key FROM email_attachments + WHERE message_id = ?`, ) .bind(input.messageId) - .all<{ storage_key: string }>() - const blobKeys = [ - ...(row ? [emailRawMimeKey(row.user_id, input.messageId)] : []), - ...(attachmentRows.results ?? []).map((result) => result.storage_key), + .all<{ storage_kind: string; storage_key: string | null }>() + const attachments = attachmentRows.results ?? [] + const capturedBlobs: Array<{ + key: string + role: 'raw_mime' | 'attachment' + }> = [ + ...(row + ? [ + { + key: emailRawMimeKey(row.user_id, input.messageId), + role: 'raw_mime' as const, + }, + ] + : []), + ...attachments.flatMap((attachment) => + attachment.storage_key == null + ? [] + : [ + { + key: attachment.storage_key, + role: 'attachment' as const, + }, + ], + ), ] // Atomic batch: a partial delete (attachments gone, message left) would // lose the storage_key values needed to delete the R2 blobs on retry. @@ -1097,10 +1149,24 @@ export async function deleteEmailMessageById(input: { .prepare(`DELETE FROM email_messages WHERE id = ?`) .bind(input.messageId), ]) - for (const blobKey of blobKeys) { - await input.blobs.delete(blobKey).catch((error: unknown) => { - console.warn('email-blob-delete-failed', blobKey, error) - }) + const blobDeletions: Array = [] + for (const captured of capturedBlobs) { + try { + await input.blobs.delete(captured.key) + blobDeletions.push({ ...captured, deleted: true }) + } catch (error: unknown) { + console.warn('email-blob-delete-failed', captured.key, error) + blobDeletions.push({ ...captured, deleted: false }) + } + } + return { + messageFound: row != null, + ownerUserId: row?.user_id ?? null, + attachmentsSeen: attachments.length, + externalAttachmentsSeen: attachments.filter( + (attachment) => attachment.storage_kind === 'external', + ).length, + blobDeletions, } } From 8a5c8e2ef033d2401e0fe37948e0bfefc6f499c5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 1 Aug 2026 21:37:16 +0000 Subject: [PATCH 39/62] fix(admin): align canary delete result contract Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 30 ++++++++++--------- .../admin-mailbox-maintenance.node.test.ts | 2 +- .../admin/admin-mailbox-maintenance.ts | 18 ++++++----- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 071d820b15..de2b4fd134 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -523,14 +523,16 @@ rows only. re-parses the resolved MIME from that blob. - Message deletes always delete the deterministic `emailRawMimeKey(userId, messageId)` from R2 (production writers always store - that canonical key). `deleteEmailMessageById` runs an atomic D1 batch - (attachments, then message), then best-effort R2 blob deletes. Live explicit - and retention deletes do not call Mailbox mirror helpers today; the parity - lane repairs DO state via purge/rebuild. Direct delete wiring is pending. User - email retention and system-email retention stay strict: blob delete before row - delete; failed blob deletes skip the row for retry. Account deletion stays - strict before atomic D1 finalization; a failed blob delete preserves every - message row for retry. + that canonical key). `deleteEmailMessageById` captures ownership + attachment + `storage_key` values, optionally enforces an `expectedUserId` owner fence, + runs an atomic D1 batch (attachments, then message), then best-effort R2 blob + deletes and returns the exact captured blob deletion inventory/outcomes for + internal verification. Live explicit and retention deletes do not call Mailbox + mirror helpers today; the parity lane repairs DO state via purge/rebuild. + Direct delete wiring is pending. User email retention and system-email + retention stay strict: blob delete before row delete; failed blob deletes skip + the row for retry. Account deletion stays strict before atomic D1 + finalization; a failed blob delete preserves every message row for retry. - Bucket names: `kody-email-blobs` (production), per-preview `{worker}-email-blobs` buckets created and cleaned up by `tools/ci/preview-resources.ts`, and the test env reuses the preview-style @@ -979,12 +981,12 @@ max 100, then status), `retention` (natural cutoffs only), and `delete_message` the last considered owner). Per-owner failures are isolated. `delete_message` takes `stable_user_id` + `message_id`, verifies ownership with -`getEmailMessageById` (missing/foreign rejected), collects attachment metadata / -storage keys without reading content, deletes through existing -`deleteEmailMessageById` (`APP_DB` + `EMAIL_BLOBS`), then verifies the D1 -message is absent and every canonical raw-MIME / external-attachment R2 object -is absent via `head`. Returns aggregate booleans/counts only (no addresses, -bodies, filenames, or keys). Audit success reason includes the target ids. +`getEmailMessageById` (missing/foreign rejected), deletes through +`deleteEmailMessageById` (`APP_DB` + `EMAIL_BLOBS`, `expectedUserId` fence), +then verifies the D1 message is absent and every exact captured blob key from +that delete inventory is absent via `head`. Returns aggregate booleans/counts +only (no addresses, bodies, filenames, or keys). Audit success reason includes +the target ids. Retention returns D1 delete/error totals plus aggregate Mailbox before/after counts (no message ids or email content). No seed or arbitrary-cutoff surface. diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts index 6c579d3785..8e00649a7f 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts @@ -101,7 +101,7 @@ const emptyDeleteResult = { externalAttachmentsSeen: 1, rawMimeBlobAbsent: true, externalAttachmentBlobsAbsent: 1, - allCanonicalBlobsAbsent: true, + allCapturedBlobsAbsent: true, } function createAdminCtx() { diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts index 925c15d155..42523edb04 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts @@ -113,26 +113,28 @@ const deleteMessageResultSchema = z.object({ .number() .int() .nonnegative() - .describe('Attachment metadata rows observed before delete.'), + .describe('Attachment metadata rows captured at delete time.'), externalAttachmentsSeen: z .number() .int() .nonnegative() - .describe('External-storage attachment rows observed before delete.'), + .describe('External-storage attachment rows captured at delete time.'), rawMimeBlobAbsent: z .boolean() - .describe('True when head(emailRawMimeKey) finds no object after delete.'), + .describe( + 'True when head() finds no object for every captured raw-MIME blob key.', + ), externalAttachmentBlobsAbsent: z .number() .int() .nonnegative() .describe( - 'Count of canonical external attachment keys with no R2 object after delete.', + 'Count of captured attachment blob keys with no R2 object after delete.', ), - allCanonicalBlobsAbsent: z + allCapturedBlobsAbsent: z .boolean() .describe( - 'True when the canonical raw-MIME key and every external attachment key are absent.', + 'True when every blob key captured by deleteEmailMessageById is absent via head.', ), }) @@ -210,7 +212,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( ...adminMutationCapabilityAccess, name: 'admin_mailbox_maintenance', description: - 'Admin-only Mailbox parity/retention/delete maintenance: aggregate status (no email content), bounded reconcileMailboxParity, keyset-paged natural retention (D1 authoritative prune, skip DO while owner still has expired D1 rows, then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget), or owner-scoped delete_message (getEmailMessageById ownership check, existing deleteEmailMessageById, post-delete D1/R2 head aggregates). Never accepts arbitrary cutoffs or seed data. Audited.', + 'Admin-only Mailbox parity/retention/delete maintenance: aggregate status (no email content), bounded reconcileMailboxParity, keyset-paged natural retention (D1 authoritative prune, skip DO while owner still has expired D1 rows, then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget), or owner-scoped delete_message (getEmailMessageById ownership check, deleteEmailMessageById with expectedUserId fence, post-delete D1/R2 head over exact captured blob keys). Never accepts arbitrary cutoffs or seed data. Audited.', keywords: [ 'admin', 'mailbox', @@ -294,7 +296,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( : 'unknown' const targetMessageId = args.action === 'delete_message' ? args.message_id : 'unknown' - return `action=delete_message;stable_user_id=${targetStableUserId};message_id=${targetMessageId};d1_absent=${result.result.d1MessageAbsent ? 1 : 0};blobs_absent=${result.result.allCanonicalBlobsAbsent ? 1 : 0}` + return `action=delete_message;stable_user_id=${targetStableUserId};message_id=${targetMessageId};d1_absent=${result.result.d1MessageAbsent ? 1 : 0};blobs_absent=${result.result.allCapturedBlobsAbsent ? 1 : 0}` } default: { const exhaustive: never = result From 3a7782007371e7609055c08a491549c3f86cf270 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 02:33:02 +0000 Subject: [PATCH 40/62] feat(email): cut owner reads over to Mailbox Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 63 +- .../architecture/feature-flags.md | 27 +- packages/worker/src/app/account-email-data.ts | 286 +++------ .../app/handlers/account-email.node.test.ts | 150 ++--- .../src/app/request-feature-flags-cache.ts | 5 + packages/worker/src/email/mailbox-do.ts | 13 + .../src/email/mailbox-do.workers.test.ts | 20 + .../email/mailbox-read-cutover.node.test.ts | 572 +++++++++++------- .../worker/src/email/mailbox-read-cutover.ts | 466 ++++++++++++-- packages/worker/src/email/mailbox-store.ts | 133 ++-- packages/worker/src/email/mailbox-types.ts | 27 + packages/worker/src/email/repo.ts | 103 ++++ packages/worker/src/email/service.ts | 55 +- packages/worker/src/feature-flags/registry.ts | 2 +- packages/worker/src/feature-flags/service.ts | 19 +- .../email/email-attachment-get.node.test.ts | 32 +- .../email/email-attachment-get.ts | 17 +- .../email-delivery-event-list.node.test.ts | 18 +- .../email/email-delivery-event-list.ts | 17 +- .../capabilities/email/email-message-get.ts | 31 +- .../email/email-message-list.node.test.ts | 18 +- .../capabilities/email/email-message-list.ts | 19 +- .../email/email-message-search.node.test.ts | 23 +- .../email/email-message-search.ts | 17 +- 24 files changed, 1425 insertions(+), 708 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e4231d95e0..e8fb5ce933 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -940,17 +940,29 @@ purge fails (timeout/error), rebuild state is **not** reset — the user stays ineligible for soak, `last_error` records the failure, and the next ticks retry purge before advancing cursors. -**Phase 3 read cutover (prepared, not flipped):** -`packages/worker/src/email/mailbox-read-cutover.ts` defines a default-off -`mailbox-read-cutover` feature flag plus `getOwnerEmailMessageById` / -`isMailboxReadCutoverEnabled`. The gate requires the flag **and** per-user -parity soak: `mailbox_parity_matching_since` at least 24h old, +**Phase 3 owner-facing read cutover (wired, flag default-off):** +`packages/worker/src/email/mailbox-read-cutover.ts` gates app `/account/email` +inbox/detail and MCP `email_message_*` / `email_attachment_get` / +`email_delivery_event_list` through Mailbox when the default-off +`mailbox-read-cutover` flag **and** per-user parity soak both pass: +`mailbox_parity_matching_since` ≥ **2h** (pre-launch; Kent-approved on ~38-user +fleet evidence — **TODO(launch-hardening):** revisit), `mailbox_parity_checked_at` fresh within 6h, -`mailbox_parity_mismatch_count === 0`, exact `stable_user_id` match, and -`deleting_at IS NULL` (deleting accounts fail closed to D1). D1 errors, missing -parity state, and an off flag fail closed to D1 reads. No live reader calls this -adapter yet; exposure recording is intentionally omitted until a live gate site -exists (see [Feature flags](./feature-flags.md)). +`mailbox_parity_mismatch_count === 0`, exact `stable_user_id` match, +`deleting_at IS NULL`. D1 dual-writes remain. Provider reverse lookup, outbound +reply/message-id lookup, inbound, and package-subscription reads stay on D1. +When the gate is on, DO errors propagate with no D1 fallback. Raw MIME / R2 +attachment bytes still load through existing blob helpers after Mailbox metadata +selection. + +**Operator enable prerequisite (before flipping the flag):** confirm a sealed DR +day covering D1 + `EMAIL_BLOBS` under +[disaster recovery](../disaster-recovery.md) — bucket `kody-production-backups` +in the DR (KCD) account, latest `daily/full//manifest.json` with verified +stored-object digest/checksum, and +`admin_mailbox_maintenance({ action: "status" })` showing `matching`/`eligible` +counts with `mismatch`/`error` at zero for the target cohort. Deploy first; then +enable per user. **Retention** is self-enforced inside the DO with alarms (`mailboxMessageRetentionDays = 365`, `mailboxDeliveryEventRetentionDays = 90`). @@ -1032,12 +1044,13 @@ backfills all owner messages and delivery events (including pre-claim bounded rejection rows), durable content-watermark replays, compares owner-scoped D1 vs Mailbox counts, persists soak state on `users` (migration `0125`), re-purges the DO when account deletion races the lane, and repairs delete drift via -purge/rebuild. Direct delete wiring is pending. Live email read/write authority -is unchanged — D1 `email_*` tables remain authoritative for all live paths. D1 -email rows and existing R2 inventory deletion stay authoritative during expand. -**Phase 2 contract completion** (every user-mail D1 mutation also writes the DO -on live paths, including retention deletes) remains pending; terminal inbound + -parity cover the high-risk live surface today. +purge/rebuild. Direct delete wiring is pending. D1 remains write authority for +all live paths; owner-facing reads may use Mailbox when the default-off +`mailbox-read-cutover` flag and parity soak pass (phase 3). D1 email rows and +existing R2 inventory deletion stay authoritative during expand. **Phase 2 +contract completion** (every user-mail D1 mutation also writes the DO on live +paths, including retention deletes) remains pending; terminal inbound + parity +cover the high-risk live surface today. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -1067,16 +1080,18 @@ parity cover the high-risk live surface today. compares, soak tracking on `users` (migration `0125`), and repairs delete drift via purge/rebuild. **Still pending for phase-2 contract completion:** direct delete wiring for explicit/retention deletes (including retention - sweeper metadata-delete mirrors). D1 authority and read cutover are unchanged - (prepared adapter exists; not wired). -3. **Reads cut over after production soak** — user-mail reads move to the DO - only after production parity soak is verified - (`mailbox_parity_matching_since` ≥ 24h continuous exact counts, fresh + sweeper metadata-delete mirrors). D1 remains write authority; owner-facing + read cutover is wired behind the default-off flag (phase 3). +3. **Owner-facing reads cut over after production soak** — app inbox/detail and + MCP list/get/search/attachment/delivery-event reads move to the DO only after + production parity soak is verified (`mailbox_parity_matching_since` ≥ 2h + continuous exact counts pre-launch — TODO(launch-hardening) revisit, fresh `mailbox_parity_checked_at`, zero `mailbox_parity_mismatch_count`, account not marked for deletion) **and** the default-off `mailbox-read-cutover` flag - is enabled per user. D1 writes continue. A prepared - `getOwnerEmailMessageById` adapter exists but is **not wired**; no live - reader has flipped and no cutover exposures are recorded yet. + is enabled per user. D1 writes continue. + Internal/inbound/provider-lookup/package-subscription readers stay on D1. + Live gate evaluations record flag exposures (session cache or cutover memo + chokepoint). 4. **D1 write-off / event retirement** — stop writing moved user-mail metadata to D1; retire dual-write and event/mirror machinery used only for the migration. diff --git a/docs/contributing/architecture/feature-flags.md b/docs/contributing/architecture/feature-flags.md index f5e0b8a776..7901a6efd8 100644 --- a/docs/contributing/architecture/feature-flags.md +++ b/docs/contributing/architecture/feature-flags.md @@ -73,14 +73,20 @@ end-to-end (`e2e/admin-feature-flags.spec.ts`). Default-off migration flag for phase 3 of the Mailbox expand/contract track (see [Data storage — Mailbox](./data-storage.md#durable-objects-mailbox)). It gates -whether owner-scoped user-mail **message reads** may come from the per-user -Mailbox Durable Object instead of D1. +whether owner-facing user-mail **reads** (app `/account/email` inbox + detail, +and MCP `email_message_list` / `get` / `search`, `email_attachment_get`, +`email_delivery_event_list`) may come from the per-user Mailbox Durable Object +instead of D1. Writes, inbound, package-subscription, and provider reverse +lookup stay on D1. Evaluation alone is not enough. A live read path must also pass the per-user parity soak on `users` (migration `0125-mailbox-parity-state.sql`): -- `mailbox_parity_matching_since` at least **24 hours** old (continuous exact - D1↔Mailbox count parity), +- `mailbox_parity_matching_since` at least **2 hours** old (continuous exact + D1↔Mailbox count parity). Pre-launch rationale: ~38-user fleet evidence with + every-5m parity plus fail-closed freshness/mismatch/deleting/identity checks + (Kent-approved). **TODO(launch-hardening):** revisit lengthening soak when the + fleet grows. - `mailbox_parity_checked_at` fresh within **6 hours** (every-5m lane with bounded batch scaling; not a strict 5m SLA), - `mailbox_parity_mismatch_count === 0`, @@ -92,14 +98,11 @@ mismatch depth, deleting accounts, and an off flag all **fail closed** to D1-authoritative reads. When the gate is on, Mailbox DO read errors propagate with no D1 fallback. -The prepared adapter lives in -`packages/worker/src/email/mailbox-read-cutover.ts` -(`isMailboxReadCutoverEnabled`, `getOwnerEmailMessageById`). **No live reader -calls it yet** — phase 3 is not flipped. Exposure recording is intentionally -omitted until a live gate site exists (the session/MCP evaluation caches, or an -explicit `recordFeatureFlagExposures` at the first switched call site). Until -then, the flag appears in admin/session evaluation maps but records no -`FLAG_EXPOSURES` rows. +Adapters live in `packages/worker/src/email/mailbox-read-cutover.ts`. The gate +records `mailbox-read-cutover` exposure once per memoized evaluation unless the +app caller already recorded flags via the session cache +(`skipExposureRecording`). Deploy with the flag default-off; operators enable +per user after backup verification (see data-storage). Success metric: `email_received` error rate should decrease after verified parity soak and cutover (`successMetric` on the registry definition). diff --git a/packages/worker/src/app/account-email-data.ts b/packages/worker/src/app/account-email-data.ts index 8a6ef3a4ff..5ace471fc4 100644 --- a/packages/worker/src/app/account-email-data.ts +++ b/packages/worker/src/app/account-email-data.ts @@ -1,43 +1,45 @@ -import { parseJsonStringArray } from '@kody-internal/shared/json-parsing.ts' +import { utcDayKey } from '@kody-internal/shared/date-keys.ts' import { type readAuthenticatedAppUser } from '#app/authenticated-user.ts' import { emailVerificationRequiredMessage, isAccountEmailVerified, } from '#app/email-verification.ts' -import { readPagination } from '#worker/query-params.ts' +import { hasResolvedRequestFeatureFlags } from '#app/request-feature-flags-cache.ts' import { toMessageDetail } from '#mcp/capabilities/email/shared.ts' import { - resolveEmailResourceLimit, - type PlanName, -} from '#worker/entitlements/plans.ts' -import { - getUserPlan, - readDailyEntitlementResourceUsage, - readEntitlementResourceUsage, -} from '#worker/entitlements/service.ts' -import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' + getOwnerEmailMessageById, + listOwnerEmailAttachmentsForMessage, + listOwnerEmailDeliveryEvents, + listOwnerEmailMessagesPage, + type MailboxReadCutoverMemo, +} from '#worker/email/mailbox-read-cutover.ts' import { buildPlatformEmailAddress, getPlatformEmailDomain, } from '#worker/email/platform-address.ts' import { - getEmailMessageById, - listEmailAttachmentsForMessage, - listEmailDeliveryEvents, listEmailInboxAddressesForUser, listEmailInboxesForUser, } from '#worker/email/repo.ts' import { emailClassificationValues, - emailDeliveryStatusValues, - emailDirectionValues, - emailProcessingStatusValues, type EmailClassification, type EmailDeliveryStatus, type EmailDirection, + type EmailMessageRecord, type EmailProcessingStatus, } from '#worker/email/types.ts' -import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { + resolveEmailResourceLimit, + type PlanName, +} from '#worker/entitlements/plans.ts' +import { + getUserPlan, + readDailyEntitlementResourceUsage, + readEntitlementResourceUsage, +} from '#worker/entitlements/service.ts' +import { type UserMeterEnv } from '#worker/entitlements/user-meter-client.ts' +import { readPagination } from '#worker/query-params.ts' type AuthenticatedUser = NonNullable< Awaited> @@ -184,41 +186,6 @@ export function readAccountEmailSelectedMessageId( return url.searchParams.get('selected')?.trim() || null } -/** Escape LIKE wildcards so user queries match literally. */ -function escapeLikePattern(value: string) { - return value.replaceAll(/[\\%_]/g, (character) => `\\${character}`) -} - -function parseDirection(value: unknown): EmailDirection { - const raw = String(value) - return (emailDirectionValues as ReadonlyArray).includes(raw) - ? (raw as EmailDirection) - : 'inbound' -} - -function parseProcessingStatus(value: unknown): EmailProcessingStatus { - const raw = String(value) - return (emailProcessingStatusValues as ReadonlyArray).includes(raw) - ? (raw as EmailProcessingStatus) - : 'stored' -} - -function parseDeliveryStatus(value: unknown): EmailDeliveryStatus | null { - if (value == null) return null - const raw = String(value) - return (emailDeliveryStatusValues as ReadonlyArray).includes(raw) - ? (raw as EmailDeliveryStatus) - : null -} - -function parseClassification(value: unknown): EmailClassification { - if (value == null) return 'accepted' - const raw = String(value) - return (emailClassificationValues as ReadonlyArray).includes(raw) - ? (raw as EmailClassification) - : 'accepted' -} - export function readAccountEmailClassificationFilter( requestUrl: string, ): EmailClassification | null { @@ -229,143 +196,34 @@ export function readAccountEmailClassificationFilter( : null } -function rowToListItem( - row: Record, -): AccountEmailMessageListItem { - return { - id: String(row['id']), - direction: parseDirection(row['direction']), - inbox_id: row['inbox_id'] == null ? null : String(row['inbox_id']), - thread_id: row['thread_id'] == null ? null : String(row['thread_id']), - from_address: - row['from_address'] == null ? null : String(row['from_address']), - envelope_from: - row['envelope_from'] == null ? null : String(row['envelope_from']), - to_addresses: parseJsonStringArray(row['to_addresses_json']), - subject: row['subject'] == null ? null : String(row['subject']), - message_id_header: - row['message_id_header'] == null - ? null - : String(row['message_id_header']), - processing_status: parseProcessingStatus(row['processing_status']), - classification: parseClassification(row['classification']), - classification_reason: - row['classification_reason'] == null - ? null - : String(row['classification_reason']), - provider_message_id: - row['provider_message_id'] == null - ? null - : String(row['provider_message_id']), - delivery_status: parseDeliveryStatus(row['delivery_status']), - delivery_status_at: - row['delivery_status_at'] == null - ? null - : String(row['delivery_status_at']), - error: row['error'] == null ? null : String(row['error']), - received_at: row['received_at'] == null ? null : String(row['received_at']), - sent_at: row['sent_at'] == null ? null : String(row['sent_at']), - created_at: String(row['created_at']), - updated_at: String(row['updated_at']), - } +function asStringAddresses(value: Array): Array { + return value.filter((entry): entry is string => typeof entry === 'string') } -async function countAndListMessages(input: { - db: D1Database - userId: string - query: string - classification: EmailClassification | null - pageSize: number - offset: number -}) { - const classification = input.classification - if (input.query) { - const pattern = `%${escapeLikePattern(input.query.toLowerCase())}%` - const [totalResult, messageRows] = await Promise.all([ - input.db - .prepare( - `SELECT COUNT(*) AS total - FROM email_messages - WHERE user_id = ? - AND (? IS NULL OR classification = ?) - AND ( - LOWER(subject) LIKE ? ESCAPE '\\' - OR LOWER(from_address) LIKE ? ESCAPE '\\' - OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' - )`, - ) - .bind( - input.userId, - classification, - classification, - pattern, - pattern, - pattern, - ) - .first<{ total: number }>(), - input.db - .prepare( - `SELECT * - FROM email_messages - WHERE user_id = ? - AND (? IS NULL OR classification = ?) - AND ( - LOWER(subject) LIKE ? ESCAPE '\\' - OR LOWER(from_address) LIKE ? ESCAPE '\\' - OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' - ) - ORDER BY created_at DESC, id DESC - LIMIT ? OFFSET ?`, - ) - .bind( - input.userId, - classification, - classification, - pattern, - pattern, - pattern, - input.pageSize, - input.offset, - ) - .all>(), - ]) - return { - total: Number(totalResult?.total ?? 0), - messages: (messageRows.results ?? []).map(rowToListItem), - } - } - - const [totalResult, messageRows] = await Promise.all([ - input.db - .prepare( - `SELECT COUNT(*) AS total - FROM email_messages - WHERE user_id = ? - AND (? IS NULL OR classification = ?)`, - ) - .bind(input.userId, classification, classification) - .first<{ total: number }>(), - input.db - .prepare( - `SELECT * - FROM email_messages - WHERE user_id = ? - AND (? IS NULL OR classification = ?) - ORDER BY created_at DESC, id DESC - LIMIT ? OFFSET ?`, - ) - .bind( - input.userId, - classification, - classification, - input.pageSize, - input.offset, - ) - .all>(), - ]) +function messageToListItem( + message: EmailMessageRecord, +): AccountEmailMessageListItem { return { - total: Number(totalResult?.total ?? 0), - messages: (messageRows.results ?? []).map(rowToListItem), + id: message.id, + direction: message.direction, + inbox_id: message.inboxId, + thread_id: message.threadId, + from_address: message.fromAddress, + envelope_from: message.envelopeFrom, + to_addresses: asStringAddresses(message.toAddresses), + subject: message.subject, + message_id_header: message.messageIdHeader, + processing_status: message.processingStatus, + classification: message.classification, + classification_reason: message.classificationReason, + provider_message_id: message.providerMessageId, + delivery_status: message.deliveryStatus, + delivery_status_at: message.deliveryStatusAt, + error: message.error, + received_at: message.receivedAt, + sent_at: message.sentAt, + created_at: message.createdAt, + updated_at: message.updatedAt, } } @@ -448,26 +306,39 @@ async function loadInboxes(input: { } async function loadSelectedMessage(input: { - db: D1Database - userId: string + env: Env + dbUserId: number + stableUserId: string messageId: string + skipExposureRecording: boolean + memo: MailboxReadCutoverMemo }): Promise { - const message = await getEmailMessageById({ - db: input.db, - userId: input.userId, + const message = await getOwnerEmailMessageById({ + env: input.env, + dbUserId: input.dbUserId, + stableUserId: input.stableUserId, messageId: input.messageId, + skipExposureRecording: input.skipExposureRecording, + memo: input.memo, }) if (!message) return null const [attachments, deliveryEvents] = await Promise.all([ - listEmailAttachmentsForMessage({ - db: input.db, + listOwnerEmailAttachmentsForMessage({ + env: input.env, + dbUserId: input.dbUserId, + stableUserId: input.stableUserId, messageId: message.id, + skipExposureRecording: input.skipExposureRecording, + memo: input.memo, }), - listEmailDeliveryEvents({ - db: input.db, - userId: input.userId, + listOwnerEmailDeliveryEvents({ + env: input.env, + dbUserId: input.dbUserId, + stableUserId: input.stableUserId, messageId: message.id, limit: deliveryEventsLimit, + skipExposureRecording: input.skipExposureRecording, + memo: input.memo, }), ]) const detail = toMessageDetail(message, attachments) @@ -570,15 +441,21 @@ export async function loadAccountEmailData(input: { input.request.url, input.pathMessageId, ) + const cutoverMemo: MailboxReadCutoverMemo = {} + const dbUserId = input.user.userId + const skipExposureRecording = hasResolvedRequestFeatureFlags(input.request) const [listResult, inboxes, usage, selectedMessage] = await Promise.all([ - countAndListMessages({ - db: input.env.APP_DB, - userId, + listOwnerEmailMessagesPage({ + env: input.env, + dbUserId, + stableUserId: userId, query, classification, pageSize, offset, + skipExposureRecording, + memo: cutoverMemo, }), loadInboxes({ db: input.env.APP_DB, userId }), loadUsage({ @@ -589,9 +466,12 @@ export async function loadAccountEmailData(input: { }), selectedMessageId ? loadSelectedMessage({ - db: input.env.APP_DB, - userId, + env: input.env, + dbUserId, + stableUserId: userId, messageId: selectedMessageId, + skipExposureRecording, + memo: cutoverMemo, }) : Promise.resolve(null), ]) @@ -607,7 +487,7 @@ export async function loadAccountEmailData(input: { }), verificationMessage: null, inboxes, - messages: listResult.messages, + messages: listResult.messages.map(messageToListItem), selectedMessage, usage, page, diff --git a/packages/worker/src/app/handlers/account-email.node.test.ts b/packages/worker/src/app/handlers/account-email.node.test.ts index 7110a388ff..4c0261799e 100644 --- a/packages/worker/src/app/handlers/account-email.node.test.ts +++ b/packages/worker/src/app/handlers/account-email.node.test.ts @@ -6,29 +6,6 @@ import type * as EntitlementService from '#worker/entitlements/service.ts' import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' -const messageRow = { - id: 'msg-1', - direction: 'inbound', - inbox_id: 'inbox-1', - thread_id: null, - from_address: 'sender@example.com', - envelope_from: 'sender@example.com', - to_addresses_json: '["user@inbox.example.com"]', - subject: 'Hello from sender', - message_id_header: '', - processing_status: 'stored', - classification: 'accepted', - classification_reason: null, - provider_message_id: null, - delivery_status: null, - delivery_status_at: null, - error: null, - received_at: new Date(0).toISOString(), - sent_at: null, - created_at: new Date(0).toISOString(), - updated_at: new Date(0).toISOString(), -} - const messageRecord = { id: 'msg-1', direction: 'inbound' as const, @@ -126,8 +103,12 @@ const mockModule = vi.hoisted(() => ({ updatedAt: new Date(0).toISOString(), }, ]), - getEmailMessageById: vi.fn(async () => messageRecord), - listEmailAttachmentsForMessage: vi.fn(async () => [ + listOwnerEmailMessagesPage: vi.fn(async () => ({ + total: 1, + messages: [messageRecord], + })), + getOwnerEmailMessageById: vi.fn(async () => messageRecord), + listOwnerEmailAttachmentsForMessage: vi.fn(async () => [ { id: 'att-1', messageId: 'msg-1', @@ -141,7 +122,7 @@ const mockModule = vi.hoisted(() => ({ createdAt: new Date(0).toISOString(), }, ]), - listEmailDeliveryEvents: vi.fn(async () => [ + listOwnerEmailDeliveryEvents: vi.fn(async () => [ { id: 'evt-1', messageId: 'msg-1', @@ -217,12 +198,17 @@ vi.mock('#worker/email/repo.ts', () => ({ mockModule.listEmailInboxesForUser(...args), listEmailInboxAddressesForUser: (...args: Array) => mockModule.listEmailInboxAddressesForUser(...args), - getEmailMessageById: (...args: Array) => - mockModule.getEmailMessageById(...args), - listEmailAttachmentsForMessage: (...args: Array) => - mockModule.listEmailAttachmentsForMessage(...args), - listEmailDeliveryEvents: (...args: Array) => - mockModule.listEmailDeliveryEvents(...args), +})) + +vi.mock('#worker/email/mailbox-read-cutover.ts', () => ({ + listOwnerEmailMessagesPage: (...args: Array) => + mockModule.listOwnerEmailMessagesPage(...args), + getOwnerEmailMessageById: (...args: Array) => + mockModule.getOwnerEmailMessageById(...args), + listOwnerEmailAttachmentsForMessage: (...args: Array) => + mockModule.listOwnerEmailAttachmentsForMessage(...args), + listOwnerEmailDeliveryEvents: (...args: Array) => + mockModule.listOwnerEmailDeliveryEvents(...args), })) vi.mock('#worker/email/service.ts', () => ({ @@ -232,39 +218,18 @@ vi.mock('#worker/email/service.ts', () => ({ const { createAccountEmailApiHandler } = await import('./account-email.ts') -function createCountResult(total: number) { - return { - bind: vi.fn().mockReturnValue({ - first: vi.fn(async () => ({ total })), - all: vi.fn(async () => ({ results: [] })), - }), - } -} - -function createListResult(rows: Array>) { - return { - bind: vi.fn().mockReturnValue({ - first: vi.fn(async () => null), - all: vi.fn(async () => ({ results: rows })), - }), - } -} - function createEnv(input?: { meter?: ReturnType - messageRows?: Array> + messages?: Array messageTotal?: number }) { const meter = input?.meter ?? createInMemoryUserMeterEnv() - const messageRows = input?.messageRows ?? [messageRow] - const messageTotal = input?.messageTotal ?? messageRows.length - mockModule.prepare.mockImplementation((query: string) => { - const normalized = String(query).replace(/\s+/g, ' ').trim().toLowerCase() - if (normalized.includes('count(*)')) { - return createCountResult(messageTotal) - } - return createListResult(messageRows) - }) + const messages = input?.messages ?? [messageRecord] + const messageTotal = input?.messageTotal ?? messages.length + mockModule.listOwnerEmailMessagesPage.mockImplementation(async () => ({ + total: messageTotal, + messages, + })) return { APP_DB: { prepare: (...args: Array) => mockModule.prepare(...args), @@ -286,9 +251,9 @@ function useFrozenUtcTime(iso: string) { } test('email API lists messages with pagination, usage, and selected detail', async () => { - mockModule.prepare.mockClear() - mockModule.getEmailMessageById.mockClear() - mockModule.listEmailDeliveryEvents.mockClear() + mockModule.listOwnerEmailMessagesPage.mockClear() + mockModule.getOwnerEmailMessageById.mockClear() + mockModule.listOwnerEmailDeliveryEvents.mockClear() using _frozenTime = useFrozenUtcTime('2026-07-31T15:00:00.000Z') const day = utcDayKey() const userId = 'stable-user-1' @@ -316,7 +281,7 @@ test('email API lists messages with pagination, usage, and selected detail', asy }) expect(listResponse.status).toBe(200) expect(listResponse.headers.get('Cache-Control')).toBe('no-store') - expect(mockModule.getEmailMessageById).not.toHaveBeenCalled() + expect(mockModule.getOwnerEmailMessageById).not.toHaveBeenCalled() await expect(listResponse.json()).resolves.toMatchObject({ ok: true, emailVerified: true, @@ -360,8 +325,7 @@ test('email API lists messages with pagination, usage, and selected detail', asy classification: null, }) - mockModule.prepare.mockClear() - mockModule.getEmailMessageById.mockClear() + mockModule.getOwnerEmailMessageById.mockClear() const selectionMeter = createInMemoryUserMeterEnv() const envWithSelection = createEnv({ meter: selectionMeter, @@ -390,14 +354,16 @@ test('email API lists messages with pagination, usage, and selected detail', asy ), }) expect(selectedResponse.status).toBe(200) - expect(mockModule.getEmailMessageById).toHaveBeenCalledWith({ - db: envWithSelection.APP_DB, - userId: 'stable-user-1', - messageId: 'msg-1', - }) - expect(mockModule.listEmailDeliveryEvents).toHaveBeenCalledWith( + expect(mockModule.getOwnerEmailMessageById).toHaveBeenCalledWith( expect.objectContaining({ - userId: 'stable-user-1', + dbUserId: 42, + stableUserId: 'stable-user-1', + messageId: 'msg-1', + }), + ) + expect(mockModule.listOwnerEmailDeliveryEvents).toHaveBeenCalledWith( + expect.objectContaining({ + stableUserId: 'stable-user-1', messageId: 'msg-1', }), ) @@ -481,23 +447,18 @@ test('email API usage reads initialized UserMeter daily counts', async () => { }) test('email API lists classification filters and classifies inbound messages', async () => { - const quarantinedRow = { - ...messageRow, + const quarantinedMessage = { + ...messageRecord, id: 'msg-quarantined', - classification: 'quarantined', - classification_reason: 'DMARC failed.', + classification: 'quarantined' as const, + classificationReason: 'DMARC failed.', } const meter = createInMemoryUserMeterEnv() const env = createEnv({ meter, - messageRows: [quarantinedRow], - }) - mockModule.getEmailMessageById.mockResolvedValueOnce({ - ...messageRecord, - id: 'msg-quarantined', - classification: 'quarantined', - classificationReason: 'DMARC failed.', + messages: [quarantinedMessage], }) + mockModule.getOwnerEmailMessageById.mockResolvedValueOnce(quarantinedMessage) const handler = createAccountEmailApiHandler(env) const listResponse = await handler.handler({ @@ -525,7 +486,8 @@ test('email API lists classification filters and classifies inbound messages', a mockModule.setEmailMessageClassification.mockClear() // Reload after classify uses the default message list again. - createEnv({ meter, messageRows: [messageRow] }) + createEnv({ meter, messages: [messageRecord] }) + mockModule.getOwnerEmailMessageById.mockResolvedValue(messageRecord) const quarantineResponse = await handler.handler({ request: new Request('https://example.com/account/email.json', { method: 'POST', @@ -616,7 +578,7 @@ test('email API gates unverified accounts and skips mailbox queries', async () = }, } as never) mockModule.isAccountEmailVerified.mockResolvedValueOnce(false) - mockModule.prepare.mockClear() + mockModule.listOwnerEmailMessagesPage.mockClear() mockModule.listEmailInboxesForUser.mockClear() mockModule.getUserPlan.mockClear() @@ -627,7 +589,7 @@ test('email API gates unverified accounts and skips mailbox queries', async () = expect(response.status).toBe(200) expect(mockModule.listEmailInboxesForUser).not.toHaveBeenCalled() expect(mockModule.getUserPlan).not.toHaveBeenCalled() - expect(mockModule.prepare).not.toHaveBeenCalled() + expect(mockModule.listOwnerEmailMessagesPage).not.toHaveBeenCalled() await expect(response.json()).resolves.toMatchObject({ ok: true, emailVerified: false, @@ -642,7 +604,7 @@ test('email API gates unverified accounts and skips mailbox queries', async () = }) test('email API scopes message detail lookups to the signed-in userId', async () => { - mockModule.getEmailMessageById.mockResolvedValueOnce(null) + mockModule.getOwnerEmailMessageById.mockResolvedValueOnce(null) const env = createEnv() const handler = createAccountEmailApiHandler(env) const response = await handler.handler({ @@ -651,11 +613,13 @@ test('email API scopes message detail lookups to the signed-in userId', async () ), }) expect(response.status).toBe(200) - expect(mockModule.getEmailMessageById).toHaveBeenCalledWith({ - db: env.APP_DB, - userId: 'stable-user-1', - messageId: 'missing-msg', - }) + expect(mockModule.getOwnerEmailMessageById).toHaveBeenCalledWith( + expect.objectContaining({ + dbUserId: 42, + stableUserId: 'stable-user-1', + messageId: 'missing-msg', + }), + ) await expect(response.json()).resolves.toMatchObject({ ok: true, selectedMessage: null, diff --git a/packages/worker/src/app/request-feature-flags-cache.ts b/packages/worker/src/app/request-feature-flags-cache.ts index de8c4563f4..3869d5942b 100644 --- a/packages/worker/src/app/request-feature-flags-cache.ts +++ b/packages/worker/src/app/request-feature-flags-cache.ts @@ -90,3 +90,8 @@ export function loadRequestFeatureFlags( } return promise } + +/** True when this request already resolved (and recorded) feature-flag exposures. */ +export function hasResolvedRequestFeatureFlags(request: Request): boolean { + return requestFeatureFlagsStore.has(request) +} diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 117e9ee2fd..e06e65f57c 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -27,6 +27,7 @@ import { type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, + type MailboxCountMessagesInput, type MailboxCountResult, type MailboxDeleteDeliveryEventInput, type MailboxDeleteMessageMetadataInput, @@ -436,6 +437,18 @@ class MailboxBase extends DurableObject implements MailboxRpc { return this.store.searchMessages(input) } + async countMessages( + input: MailboxCountMessagesInput, + ): Promise<{ total: number }> { + return this.store.countMessages(input) + } + + async getAttachment(input: { + attachmentId: string + }): Promise { + return this.store.getAttachment(input.attachmentId) + } + async listAttachmentsForMessage(input: { messageId: string }): Promise> { diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index b1f5aa4c10..a753e9ae4b 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -96,6 +96,16 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', storageKind: 'external', }), ]) + expect( + await mailboxA.getAttachment({ attachmentId: attachment.id }), + ).toMatchObject({ + id: attachment.id, + messageId: message.id, + storageKind: 'external', + }) + expect( + await mailboxA.getAttachment({ attachmentId: 'missing-att' }), + ).toBeNull() expect( await mailboxA.getMessageByMessageIdHeader({ messageIdHeader: message.messageIdHeader!, @@ -109,6 +119,13 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', expect( (await mailboxA.searchMessages({ query: 'nope' })).messages, ).toHaveLength(0) + expect(await mailboxA.countMessages({})).toEqual({ total: 1 }) + expect( + await mailboxA.countMessages({ + query: 'project', + classification: 'accepted', + }), + ).toEqual({ total: 1 }) await mailboxB.mirrorMessage({ ownerId: ownerB, @@ -121,6 +138,9 @@ test('Mailbox mirrors, reads, searches, isolates owners, and stays idempotent', }) expect(await mailboxB.getMessage({ messageId: message.id })).toBeNull() expect(await mailboxA.getMessage({ messageId: 'msg-b' })).toBeNull() + expect( + await mailboxB.getAttachment({ attachmentId: attachment.id }), + ).toBeNull() expect(await mailboxA.countMailbox()).toMatchObject({ threads: 1, messages: 1, diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts index ad36092808..9df07d6c06 100644 --- a/packages/worker/src/email/mailbox-read-cutover.node.test.ts +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -1,22 +1,73 @@ import { expect, test, vi } from 'vitest' import { + getOwnerEmailAttachmentById, getOwnerEmailMessageById, isMailboxReadCutoverEnabled, + listOwnerEmailDeliveryEvents, + listOwnerEmailMessages, + listOwnerEmailMessagesPage, + mailboxAttachmentToEmailAttachmentRecord, mailboxMessageToEmailMessageRecord, mailboxReadCutoverCheckedAtMaxAgeMs, mailboxReadCutoverFlagKey, mailboxReadCutoverSoakMs, + searchOwnerEmailMessages, + type MailboxReadCutoverMemo, } from './mailbox-read-cutover.ts' -import { type MailboxMessageRecord } from './mailbox-types.ts' +import { + type MailboxAttachmentRecord, + type MailboxMessageRecord, +} from './mailbox-types.ts' import { type EmailMessageRecord } from './types.ts' -const { getEmailMessageByIdMock } = vi.hoisted(() => ({ +const { + getEmailMessageByIdMock, + listEmailMessagesMock, + searchEmailMessagesMock, + listEmailMessagesPageForUserMock, + listEmailAttachmentsForMessageMock, + getEmailAttachmentRecordByIdMock, + listEmailDeliveryEventsMock, + loadEmailAttachmentContentMock, + recordFeatureFlagExposuresMock, +} = vi.hoisted(() => ({ getEmailMessageByIdMock: vi.fn(), + listEmailMessagesMock: vi.fn(), + searchEmailMessagesMock: vi.fn(), + listEmailMessagesPageForUserMock: vi.fn(), + listEmailAttachmentsForMessageMock: vi.fn(), + getEmailAttachmentRecordByIdMock: vi.fn(), + listEmailDeliveryEventsMock: vi.fn(), + loadEmailAttachmentContentMock: vi.fn(), + recordFeatureFlagExposuresMock: vi.fn(), })) vi.mock('./repo.ts', () => ({ getEmailMessageById: (...args: Array) => getEmailMessageByIdMock(...args), + listEmailMessages: (...args: Array) => + listEmailMessagesMock(...args), + searchEmailMessages: (...args: Array) => + searchEmailMessagesMock(...args), + listEmailMessagesPageForUser: (...args: Array) => + listEmailMessagesPageForUserMock(...args), + listEmailAttachmentsForMessage: (...args: Array) => + listEmailAttachmentsForMessageMock(...args), + getEmailAttachmentRecordById: (...args: Array) => + getEmailAttachmentRecordByIdMock(...args), + listEmailDeliveryEvents: (...args: Array) => + listEmailDeliveryEventsMock(...args), +})) + +vi.mock('./service.ts', () => ({ + loadEmailAttachmentContent: (...args: Array) => + loadEmailAttachmentContentMock(...args), + loadRawMime: vi.fn(), +})) + +vi.mock('#worker/feature-flags/exposure.ts', () => ({ + recordFeatureFlagExposures: (...args: Array) => + recordFeatureFlagExposuresMock(...args), })) const nowIso = '2026-08-01T12:00:00.000Z' @@ -160,15 +211,15 @@ function baseEmailMessage( function soakedParity(stableUserId = 'user-aaa'): ParityRow { return { stable_user_id: stableUserId, - mailbox_parity_matching_since: hoursAgoIso(30), + mailbox_parity_matching_since: hoursAgoIso(3), mailbox_parity_checked_at: hoursAgoIso(1), mailbox_parity_mismatch_count: 0, } } -function fakeMailboxEnv(getMessage: ReturnType) { +function fakeMailboxEnv(methods: Record>) { const idFromName = vi.fn((name: string) => name as unknown as DurableObjectId) - const get = vi.fn(() => ({ getMessage })) + const get = vi.fn(() => methods) return { env: { MAILBOX: { @@ -178,31 +229,114 @@ function fakeMailboxEnv(getMessage: ReturnType) { }, idFromName, get, + methods, } } -test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch parity', async () => { +test('mailbox-read-cutover soak is 2h; prior pass and young fail', async () => { + expect(mailboxReadCutoverSoakMs).toBe(2 * 60 * 60 * 1000) + const dbUserId = 7 + const stableUserId = 'user-aaa' + + const youngDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([ + [ + dbUserId, + { + ...soakedParity(), + mailbox_parity_matching_since: hoursAgoIso(1), + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + env: { APP_DB: youngDb }, + dbUserId, + stableUserId, + now: nowIso, + skipExposureRecording: true, + }), + ).resolves.toBe(false) + + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + await expect( + isMailboxReadCutoverEnabled({ + env: { APP_DB: readyDb }, + dbUserId, + stableUserId, + now: nowIso, + skipExposureRecording: true, + }), + ).resolves.toBe(true) + + const boundaryDb = createCutoverTestDb({ + flags: { overrideEnabled: null, globalEnabled: true }, + parityByUserId: new Map([ + [ + dbUserId, + { + stable_user_id: stableUserId, + mailbox_parity_matching_since: new Date( + nowMs - mailboxReadCutoverSoakMs, + ).toISOString(), + mailbox_parity_checked_at: new Date( + nowMs - mailboxReadCutoverCheckedAtMaxAgeMs, + ).toISOString(), + mailbox_parity_mismatch_count: 0, + }, + ], + ]), + }) + await expect( + isMailboxReadCutoverEnabled({ + env: { APP_DB: boundaryDb }, + dbUserId, + stableUserId, + now: nowIso, + skipExposureRecording: true, + }), + ).resolves.toBe(true) +}) + +test('mailbox-read-cutover defaults off and keeps fresh6h/mismatch/error/deleting/identity fail closed', async () => { const dbUserId = 7 const stableUserId = 'user-aaa' const d1Message = baseEmailMessage() getEmailMessageByIdMock.mockResolvedValue(d1Message) + recordFeatureFlagExposuresMock.mockClear() - // Registry default: flag off → D1, even with soaked parity. const defaultOffDb = createCutoverTestDb({ flags: { overrideEnabled: null, globalEnabled: null }, parityByUserId: new Map([[dbUserId, soakedParity()]]), }) await expect( isMailboxReadCutoverEnabled({ - db: defaultOffDb, + env: { APP_DB: defaultOffDb }, dbUserId, stableUserId, now: nowIso, }), ).resolves.toBe(false) + expect(recordFeatureFlagExposuresMock).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + stableUserId, + evaluations: { + [mailboxReadCutoverFlagKey]: { + enabled: false, + source: 'default', + }, + }, + }), + ) const getMessage = vi.fn(async () => baseMailboxMessage()) - const { env: mailboxEnv, idFromName } = fakeMailboxEnv(getMessage) + const { env: mailboxEnv, idFromName } = fakeMailboxEnv({ getMessage }) await expect( getOwnerEmailMessageById({ env: { ...mailboxEnv, APP_DB: defaultOffDb }, @@ -210,6 +344,7 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch stableUserId, messageId: 'msg-1', now: nowIso, + skipExposureRecording: true, }), ).resolves.toEqual(d1Message) expect(getEmailMessageByIdMock).toHaveBeenCalledWith({ @@ -220,59 +355,6 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch expect(getMessage).not.toHaveBeenCalled() expect(idFromName).not.toHaveBeenCalled() - // Override/global on but no parity → still D1. - for (const flags of [ - { overrideEnabled: true as boolean | null, globalEnabled: null }, - { overrideEnabled: null, globalEnabled: true as boolean | null }, - ]) { - getEmailMessageByIdMock.mockClear() - const noParityDb = createCutoverTestDb({ - flags, - parityByUserId: new Map([[dbUserId, null]]), - }) - await expect( - isMailboxReadCutoverEnabled({ - db: noParityDb, - dbUserId, - stableUserId, - now: nowIso, - }), - ).resolves.toBe(false) - await expect( - getOwnerEmailMessageById({ - env: { ...mailboxEnv, APP_DB: noParityDb }, - dbUserId, - stableUserId, - messageId: 'msg-1', - now: nowIso, - }), - ).resolves.toEqual(d1Message) - expect(getEmailMessageByIdMock).toHaveBeenCalled() - } - - // Young matching_since (< 24h soak). - const youngDb = createCutoverTestDb({ - flags: { overrideEnabled: true, globalEnabled: null }, - parityByUserId: new Map([ - [ - dbUserId, - { - ...soakedParity(), - mailbox_parity_matching_since: hoursAgoIso(23), - }, - ], - ]), - }) - await expect( - isMailboxReadCutoverEnabled({ - db: youngDb, - dbUserId, - stableUserId, - now: nowIso, - }), - ).resolves.toBe(false) - - // Stale checked_at (> 6h freshness window). const staleDb = createCutoverTestDb({ flags: { overrideEnabled: true, globalEnabled: null }, parityByUserId: new Map([ @@ -287,14 +369,14 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch }) await expect( isMailboxReadCutoverEnabled({ - db: staleDb, + env: { APP_DB: staleDb }, dbUserId, stableUserId, now: nowIso, + skipExposureRecording: true, }), ).resolves.toBe(false) - // Mismatch count blocks cutover. const mismatchedDb = createCutoverTestDb({ flags: { overrideEnabled: true, globalEnabled: null }, parityByUserId: new Map([ @@ -309,127 +391,251 @@ test('mailbox-read-cutover defaults off and requires soak + fresh zero-mismatch }) await expect( isMailboxReadCutoverEnabled({ - db: mismatchedDb, + env: { APP_DB: mismatchedDb }, dbUserId, stableUserId, now: nowIso, + skipExposureRecording: true, }), ).resolves.toBe(false) - // Cross-user parity row denied (stable_user_id must match exactly). const crossUserDb = createCutoverTestDb({ flags: { overrideEnabled: true, globalEnabled: null }, parityByUserId: new Map([[dbUserId, soakedParity('user-other')]]), }) await expect( isMailboxReadCutoverEnabled({ - db: crossUserDb, + env: { APP_DB: crossUserDb }, dbUserId, stableUserId, now: nowIso, + skipExposureRecording: true, }), ).resolves.toBe(false) - // Soaked + fresh + flag on → Mailbox DO; maps userId from the caller. - const readyDb = createCutoverTestDb({ + const deletingDb = createCutoverTestDb({ flags: { overrideEnabled: true, globalEnabled: null }, parityByUserId: new Map([[dbUserId, soakedParity()]]), + deletingUserIds: new Set([dbUserId]), }) await expect( isMailboxReadCutoverEnabled({ - db: readyDb, + env: { APP_DB: deletingDb }, dbUserId, stableUserId, now: nowIso, + skipExposureRecording: true, }), - ).resolves.toBe(true) + ).resolves.toBe(false) +}) - getEmailMessageByIdMock.mockClear() +test('mailbox-read-cutover DO on path covers list/search/get/attachment/events without D1 fallback', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + const memo: MailboxReadCutoverMemo = {} const doMessage = baseMailboxMessage({ subject: 'From DO' }) - const doGetMessage = vi.fn(async () => doMessage) - const doEnv = fakeMailboxEnv(doGetMessage) + const attachment: MailboxAttachmentRecord = { + id: 'att-1', + messageId: 'msg-1', + filename: 'note.txt', + contentType: 'text/plain', + contentId: null, + disposition: 'attachment', + size: 4, + storageKind: 'raw-mime', + storageKey: null, + createdAt: '2026-07-01T12:00:00.000Z', + } + const methods = { + getMessage: vi.fn(async () => doMessage), + listMessages: vi.fn(async () => ({ + messages: [doMessage], + nextCursor: null, + })), + searchMessages: vi.fn(async () => ({ messages: [doMessage] })), + countMessages: vi.fn(async () => ({ total: 1 })), + getAttachment: vi.fn(async () => attachment), + listAttachmentsForMessage: vi.fn(async () => [attachment]), + listDeliveryEvents: vi.fn(async () => [ + { + id: 'evt-1', + messageId: 'msg-1', + inboxId: 'inbox-1', + eventType: 'received' as const, + provider: 'cloudflare', + providerMessageId: null, + providerEventId: null, + detailJson: '{}', + needsEffectReconcile: false, + state: null, + fingerprint: null, + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: null, + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: null, + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: '2026-07-01T12:00:00.000Z', + updatedAt: '2026-07-01T12:00:00.000Z', + }, + ]), + } + const doEnv = fakeMailboxEnv(methods) + const env = { ...doEnv.env, APP_DB: readyDb } + const base = { + env, + dbUserId, + stableUserId, + now: nowIso, + memo, + skipExposureRecording: true, + } + + getEmailMessageByIdMock.mockClear() + listEmailMessagesMock.mockClear() + searchEmailMessagesMock.mockClear() + listEmailMessagesPageForUserMock.mockClear() + listEmailDeliveryEventsMock.mockClear() + getEmailAttachmentRecordByIdMock.mockClear() + loadEmailAttachmentContentMock.mockResolvedValue({ + ...mailboxAttachmentToEmailAttachmentRecord(attachment), + contentBase64: 'YmxvYg==', + content: null, + }) + + await expect(listOwnerEmailMessages({ ...base, limit: 10 })).resolves.toEqual( + [mailboxMessageToEmailMessageRecord(doMessage, stableUserId)], + ) await expect( - getOwnerEmailMessageById({ - env: { ...doEnv.env, APP_DB: readyDb }, - dbUserId, - stableUserId, - messageId: 'msg-1', - now: nowIso, + searchOwnerEmailMessages({ ...base, query: 'Hello', limit: 10 }), + ).resolves.toEqual([ + mailboxMessageToEmailMessageRecord(doMessage, stableUserId), + ]) + await expect( + listOwnerEmailMessagesPage({ + ...base, + query: '', + classification: null, + pageSize: 10, + offset: 0, }), + ).resolves.toEqual({ + total: 1, + messages: [mailboxMessageToEmailMessageRecord(doMessage, stableUserId)], + }) + await expect( + getOwnerEmailMessageById({ ...base, messageId: 'msg-1' }), ).resolves.toEqual( mailboxMessageToEmailMessageRecord(doMessage, stableUserId), ) - expect(doEnv.idFromName).toHaveBeenCalledWith(stableUserId) - expect(doGetMessage).toHaveBeenCalledWith({ messageId: 'msg-1' }) - expect(getEmailMessageByIdMock).not.toHaveBeenCalled() - - // Boundary: matching_since exactly soak age and checked_at exactly max age. - const boundaryDb = createCutoverTestDb({ - flags: { overrideEnabled: null, globalEnabled: true }, - parityByUserId: new Map([ - [ - dbUserId, - { - stable_user_id: stableUserId, - mailbox_parity_matching_since: new Date( - nowMs - mailboxReadCutoverSoakMs, - ).toISOString(), - mailbox_parity_checked_at: new Date( - nowMs - mailboxReadCutoverCheckedAtMaxAgeMs, - ).toISOString(), - mailbox_parity_mismatch_count: 0, - }, - ], - ]), + const events = await listOwnerEmailDeliveryEvents({ + ...base, + messageId: 'msg-1', + limit: 5, }) + expect(events).toEqual([ + expect.objectContaining({ + id: 'evt-1', + userId: stableUserId, + messageId: 'msg-1', + eventType: 'received', + }), + ]) + expect(methods.listDeliveryEvents).toHaveBeenCalled() await expect( - isMailboxReadCutoverEnabled({ - db: boundaryDb, - dbUserId, - stableUserId, - now: nowIso, + getOwnerEmailAttachmentById({ + ...base, + blobs: {} as R2Bucket, + attachmentId: 'att-1', }), - ).resolves.toBe(true) + ).resolves.toMatchObject({ id: 'att-1', contentBase64: 'YmxvYg==' }) + + expect(doEnv.idFromName).toHaveBeenCalledWith(stableUserId) + expect(getEmailMessageByIdMock).not.toHaveBeenCalled() + expect(listEmailMessagesMock).not.toHaveBeenCalled() + expect(searchEmailMessagesMock).not.toHaveBeenCalled() + expect(listEmailMessagesPageForUserMock).not.toHaveBeenCalled() + expect(listEmailDeliveryEventsMock).not.toHaveBeenCalled() + expect(getEmailAttachmentRecordByIdMock).not.toHaveBeenCalled() + expect(loadEmailAttachmentContentMock).toHaveBeenCalled() }) -test('mailbox-read-cutover keeps deleting accounts on D1 even when otherwise eligible', async () => { +test('mailbox-read-cutover does not fall back from DO and isolates cross-owner identity', async () => { const dbUserId = 7 const stableUserId = 'user-aaa' - const d1Message = baseEmailMessage() - getEmailMessageByIdMock.mockResolvedValue(d1Message) - - const deletingDb = createCutoverTestDb({ + const readyDb = createCutoverTestDb({ flags: { overrideEnabled: true, globalEnabled: null }, parityByUserId: new Map([[dbUserId, soakedParity()]]), - deletingUserIds: new Set([dbUserId]), }) + const failingGet = vi.fn(async () => { + throw new Error('mailbox unavailable') + }) + const failingEnv = fakeMailboxEnv({ getMessage: failingGet }) + getEmailMessageByIdMock.mockClear() await expect( - isMailboxReadCutoverEnabled({ - db: deletingDb, + getOwnerEmailMessageById({ + env: { ...failingEnv.env, APP_DB: readyDb }, dbUserId, stableUserId, + messageId: 'msg-1', now: nowIso, + skipExposureRecording: true, }), - ).resolves.toBe(false) + ).rejects.toThrow(/mailbox unavailable/) + expect(getEmailMessageByIdMock).not.toHaveBeenCalled() - const getMessage = vi.fn(async () => baseMailboxMessage()) - const { env, idFromName } = fakeMailboxEnv(getMessage) + const parityErrorDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, 'error']]), + }) + const d1Message = baseEmailMessage() + getEmailMessageByIdMock.mockResolvedValue(d1Message) await expect( getOwnerEmailMessageById({ - env: { ...env, APP_DB: deletingDb }, + env: { + ...fakeMailboxEnv({ getMessage: vi.fn() }).env, + APP_DB: parityErrorDb, + }, dbUserId, stableUserId, messageId: 'msg-1', now: nowIso, + skipExposureRecording: true, }), ).resolves.toEqual(d1Message) - expect(getEmailMessageByIdMock).toHaveBeenCalledWith({ - db: deletingDb, - userId: stableUserId, - messageId: 'msg-1', - }) - expect(getMessage).not.toHaveBeenCalled() - expect(idFromName).not.toHaveBeenCalled() + + await expect( + isMailboxReadCutoverEnabled({ + env: { APP_DB: readyDb }, + dbUserId, + stableUserId: 'user-other', + now: nowIso, + skipExposureRecording: true, + }), + ).resolves.toBe(false) }) test('mailbox-read-cutover fails closed on invalid now and future parity timestamps', async () => { @@ -442,18 +648,20 @@ test('mailbox-read-cutover fails closed on invalid now and future parity timesta await expect( isMailboxReadCutoverEnabled({ - db: readyDb, + env: { APP_DB: readyDb }, dbUserId, stableUserId, now: 'not-a-timestamp', + skipExposureRecording: true, }), ).resolves.toBe(false) await expect( isMailboxReadCutoverEnabled({ - db: readyDb, + env: { APP_DB: readyDb }, dbUserId, stableUserId, now: new Date(Number.NaN), + skipExposureRecording: true, }), ).resolves.toBe(false) @@ -471,89 +679,13 @@ test('mailbox-read-cutover fails closed on invalid now and future parity timesta }) await expect( isMailboxReadCutoverEnabled({ - db: futureCheckedDb, + env: { APP_DB: futureCheckedDb }, dbUserId, stableUserId, now: nowIso, + skipExposureRecording: true, }), ).resolves.toBe(false) - - const futureMatchingDb = createCutoverTestDb({ - flags: { overrideEnabled: true, globalEnabled: null }, - parityByUserId: new Map([ - [ - dbUserId, - { - ...soakedParity(), - mailbox_parity_matching_since: hoursFromNowIso(1), - mailbox_parity_checked_at: hoursAgoIso(1), - }, - ], - ]), - }) - await expect( - isMailboxReadCutoverEnabled({ - db: futureMatchingDb, - dbUserId, - stableUserId, - now: nowIso, - }), - ).resolves.toBe(false) -}) - -test('mailbox-read-cutover fails closed on D1 parity errors and does not fall back from DO', async () => { - const dbUserId = 7 - const stableUserId = 'user-aaa' - const d1Message = baseEmailMessage() - getEmailMessageByIdMock.mockResolvedValue(d1Message) - - const parityErrorDb = createCutoverTestDb({ - flags: { overrideEnabled: true, globalEnabled: null }, - parityByUserId: new Map([[dbUserId, 'error']]), - }) - await expect( - isMailboxReadCutoverEnabled({ - db: parityErrorDb, - dbUserId, - stableUserId, - now: nowIso, - }), - ).resolves.toBe(false) - const getMessage = vi.fn(async () => baseMailboxMessage()) - const { env } = fakeMailboxEnv(getMessage) - await expect( - getOwnerEmailMessageById({ - env: { ...env, APP_DB: parityErrorDb }, - dbUserId, - stableUserId, - messageId: 'msg-1', - now: nowIso, - }), - ).resolves.toEqual(d1Message) - expect(getMessage).not.toHaveBeenCalled() - expect(getEmailMessageByIdMock).toHaveBeenCalled() - - // When cutover is on, DO errors propagate — no silent D1 fallback. - getEmailMessageByIdMock.mockClear() - const readyDb = createCutoverTestDb({ - flags: { overrideEnabled: true, globalEnabled: null }, - parityByUserId: new Map([[dbUserId, soakedParity()]]), - }) - const failingGet = vi.fn(async () => { - throw new Error('mailbox unavailable') - }) - const failingEnv = fakeMailboxEnv(failingGet) - await expect( - getOwnerEmailMessageById({ - env: { ...failingEnv.env, APP_DB: readyDb }, - dbUserId, - stableUserId, - messageId: 'msg-1', - now: nowIso, - }), - ).rejects.toThrow(/mailbox unavailable/) - expect(failingGet).toHaveBeenCalled() - expect(getEmailMessageByIdMock).not.toHaveBeenCalled() }) test('mailboxMessageToEmailMessageRecord maps DO row with supplied stable userId', () => { @@ -570,3 +702,29 @@ test('mailboxMessageToEmailMessageRecord maps DO row with supplied stable userId }) expect(mapped).not.toHaveProperty('ownerId') }) + +test('gate memo records exposure once', async () => { + const dbUserId = 7 + const stableUserId = 'user-aaa' + const readyDb = createCutoverTestDb({ + flags: { overrideEnabled: true, globalEnabled: null }, + parityByUserId: new Map([[dbUserId, soakedParity()]]), + }) + recordFeatureFlagExposuresMock.mockClear() + const memo: MailboxReadCutoverMemo = {} + await isMailboxReadCutoverEnabled({ + env: { APP_DB: readyDb }, + dbUserId, + stableUserId, + now: nowIso, + memo, + }) + await isMailboxReadCutoverEnabled({ + env: { APP_DB: readyDb }, + dbUserId, + stableUserId, + now: nowIso, + memo, + }) + expect(recordFeatureFlagExposuresMock).toHaveBeenCalledTimes(1) +}) diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts index 40ad9fbb81..133240b1c0 100644 --- a/packages/worker/src/email/mailbox-read-cutover.ts +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -1,5 +1,5 @@ /** - * Default-off, parity-gated Mailbox read cutover (prepared; not wired). + * Default-off, parity-gated Mailbox read cutover for owner-facing reads. * * Gate requires both: * 1. feature flag `mailbox-read-cutover` enabled for the numeric `users.id` @@ -12,20 +12,60 @@ * to D1-authoritative reads. When the gate is on, Mailbox DO errors propagate * with no D1 fallback (fallback would hide parity bugs). * - * Exposure recording is intentionally omitted here: no live caller uses this - * adapter yet. Future read-path wiring owns the exposure chokepoint (the - * existing session/MCP evaluation caches once a live gate evaluates the flag, - * or an explicit `recordFeatureFlagExposures` at the first switched call site). + * Soak window is pre-launch 2h (not 24h): continuous exact D1↔Mailbox counts + * across the ~38-user fleet plus every-5m parity and fail-closed freshness / + * mismatch / deleting / identity checks were Kent-approved as sufficient before + * launch. TODO(launch-hardening): revisit extending soak when the fleet grows. + * + * Live adapters record `mailbox-read-cutover` exposure once per memoized gate + * evaluation. App callers that already ran the session flag cache pass + * `skipExposureRecording: true` so exposures are not duplicated. */ -import { isFeatureEnabled } from '#worker/feature-flags/service.ts' +import { + recordFeatureFlagExposures, + type FeatureFlagExposureEnv, +} from '#worker/feature-flags/exposure.ts' +import { evaluateFeatureFlag } from '#worker/feature-flags/service.ts' import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' -import { type MailboxMessageRecord } from './mailbox-types.ts' -import { getEmailMessageById } from './repo.ts' -import { type EmailMessageRecord } from './types.ts' +import { + type MailboxAttachmentRecord, + type MailboxDeliveryEventRecord, + type MailboxMessageRecord, +} from './mailbox-types.ts' +import { + getEmailAttachmentRecordById, + getEmailMessageById, + listEmailAttachmentsForMessage, + listEmailDeliveryEvents, + listEmailMessages, + listEmailMessagesPageForUser, + searchEmailMessages, +} from './repo.ts' +import { loadEmailAttachmentContent, loadRawMime } from './service.ts' +import { + type EmailAttachmentRecord, + type EmailClassification, + type EmailDeliveryEventRecord, + type EmailDeliveryEventType, + type EmailDeliveryStatus, + type EmailDirection, + type EmailMessageRecord, + type EmailProcessingStatus, +} from './types.ts' -/** Continuous exact parity must hold at least this long before cutover. */ -export const mailboxReadCutoverSoakMs = 24 * 60 * 60 * 1000 +/** + * Continuous exact parity must hold at least this long before cutover. + * + * Pre-launch evidence (Kent-approved): ~38-user fleet with continuous exact + * D1↔Mailbox counts, every-5m parity lane, and fail-closed freshness / + * mismatch / deleting / identity checks. 2h is intentional for pre-launch + * cutover speed — not a post-launch default. + * + * TODO(launch-hardening): revisit lengthening this soak before/at broader + * launch when fleet size and mail volume invalidate the 38-user evidence. + */ +export const mailboxReadCutoverSoakMs = 2 * 60 * 60 * 1000 /** * Parity `checked_at` must be newer than this. Sized for the every-5m parity @@ -42,8 +82,15 @@ type MailboxParityStateRow = { mailbox_parity_mismatch_count: number } -export type MailboxReadCutoverEnv = MailboxEnv & { - APP_DB: D1Database +export type MailboxReadCutoverEnv = MailboxEnv & + FeatureFlagExposureEnv & { + APP_DB: D1Database + EMAIL_BLOBS?: R2Bucket + } + +/** Caller-owned memo so one request/handler records exposure at most once. */ +export type MailboxReadCutoverMemo = { + gatePromise?: Promise } /** @@ -60,34 +107,47 @@ function resolveNowMs(now?: Date | string): number | null { return Number.isFinite(parsed) ? parsed : null } -/** - * Fail-closed cutover eligibility for one account. - * Never throws: D1 / flag evaluation failures return false. - */ -export async function isMailboxReadCutoverEnabled(input: { - db: D1Database +async function resolveDbUserId( + db: D1Database, + stableUserId: string, +): Promise { + const row = await db + .prepare(`SELECT id FROM users WHERE stable_user_id = ?`) + .bind(stableUserId) + .first<{ id: number }>() + return row?.id ?? null +} + +async function evaluateCutoverGate(input: { + env: MailboxReadCutoverEnv dbUserId: number stableUserId: string now?: Date | string + skipExposureRecording?: boolean }): Promise { try { - const flagEnabled = await isFeatureEnabled( - input.db, + const flagEvaluation = await evaluateFeatureFlag( + input.env.APP_DB, mailboxReadCutoverFlagKey, input.dbUserId, ) - if (!flagEnabled) return false + if (!input.skipExposureRecording) { + await recordFeatureFlagExposures(input.env, { + stableUserId: input.stableUserId, + evaluations: { [mailboxReadCutoverFlagKey]: flagEvaluation }, + }) + } + if (!flagEvaluation.enabled) return false - const row = await input.db - .prepare( - `SELECT stable_user_id, + const row = await input.env.APP_DB.prepare( + `SELECT stable_user_id, mailbox_parity_matching_since, mailbox_parity_checked_at, mailbox_parity_mismatch_count FROM users WHERE id = ? AND deleting_at IS NULL`, - ) + ) .bind(input.dbUserId) .first() @@ -122,6 +182,29 @@ export async function isMailboxReadCutoverEnabled(input: { } } +/** + * Fail-closed cutover eligibility for one account. + * Never throws: D1 / flag evaluation failures return false. + * Records flag exposure once per memo (unless request session cache already did). + */ +export async function isMailboxReadCutoverEnabled(input: { + env: MailboxReadCutoverEnv + dbUserId: number + stableUserId: string + now?: Date | string + /** + * When true, skip FLAG_EXPOSURES write (caller already recorded via the + * session/MCP evaluation cache). + */ + skipExposureRecording?: boolean + memo?: MailboxReadCutoverMemo +}): Promise { + if (input.memo?.gatePromise) return input.memo.gatePromise + const promise = evaluateCutoverGate(input) + if (input.memo) input.memo.gatePromise = promise + return promise +} + /** Map a Mailbox DO message row back to the D1 `EmailMessageRecord` shape. */ export function mailboxMessageToEmailMessageRecord( message: MailboxMessageRecord, @@ -164,24 +247,59 @@ export function mailboxMessageToEmailMessageRecord( } } -/** - * Prepared owner-scoped message get. Selects D1 or Mailbox via the cutover - * gate. Not wired into existing readers in this PR. - */ -export async function getOwnerEmailMessageById(input: { +export function mailboxAttachmentToEmailAttachmentRecord( + attachment: MailboxAttachmentRecord, +): EmailAttachmentRecord { + return { + id: attachment.id, + messageId: attachment.messageId, + filename: attachment.filename, + contentType: attachment.contentType, + contentId: attachment.contentId, + disposition: attachment.disposition, + size: attachment.size, + storageKind: attachment.storageKind, + storageKey: attachment.storageKey, + createdAt: attachment.createdAt, + } +} + +export function mailboxDeliveryEventToEmailDeliveryEventRecord( + event: MailboxDeliveryEventRecord, + userId: string, +): EmailDeliveryEventRecord { + return { + id: event.id, + messageId: event.messageId, + userId, + inboxId: event.inboxId, + eventType: event.eventType, + provider: event.provider, + providerMessageId: event.providerMessageId, + providerEventId: event.providerEventId, + detailJson: event.detailJson, + createdAt: event.createdAt, + } +} + +type OwnerReadBase = { env: MailboxReadCutoverEnv dbUserId: number stableUserId: string - messageId: string now?: Date | string -}): Promise { - const useMailbox = await isMailboxReadCutoverEnabled({ - db: input.env.APP_DB, - dbUserId: input.dbUserId, - stableUserId: input.stableUserId, - now: input.now, - }) + skipExposureRecording?: boolean + memo?: MailboxReadCutoverMemo +} + +async function useMailboxReads(input: OwnerReadBase): Promise { + return isMailboxReadCutoverEnabled(input) +} +/** Owner-scoped message get. Selects D1 or Mailbox via the cutover gate. */ +export async function getOwnerEmailMessageById( + input: OwnerReadBase & { messageId: string }, +): Promise { + const useMailbox = await useMailboxReads(input) if (!useMailbox) { return getEmailMessageById({ db: input.env.APP_DB, @@ -198,3 +316,273 @@ export async function getOwnerEmailMessageById(input: { if (!message) return null return mailboxMessageToEmailMessageRecord(message, input.stableUserId) } + +export async function listOwnerEmailMessages( + input: OwnerReadBase & { + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + classification?: EmailClassification | null + limit: number + }, +): Promise> { + const useMailbox = await useMailboxReads(input) + if (!useMailbox) { + return listEmailMessages({ + db: input.env.APP_DB, + userId: input.stableUserId, + inboxId: input.inboxId, + direction: input.direction, + processingStatus: input.processingStatus, + deliveryStatus: input.deliveryStatus, + classification: input.classification, + limit: input.limit, + }) + } + + const listed = await mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }).listMessages({ + inboxId: input.inboxId, + direction: input.direction, + processingStatus: input.processingStatus, + deliveryStatus: input.deliveryStatus, + classification: input.classification, + limit: input.limit, + }) + return listed.messages.map((message) => + mailboxMessageToEmailMessageRecord(message, input.stableUserId), + ) +} + +export async function searchOwnerEmailMessages( + input: OwnerReadBase & { + query: string + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + classification?: EmailClassification | null + limit: number + }, +): Promise> { + const useMailbox = await useMailboxReads(input) + if (!useMailbox) { + // D1 search has no classification filter; callers that need it use + // listOwnerEmailMessagesPage (account inbox) or list with filters. + return searchEmailMessages({ + db: input.env.APP_DB, + userId: input.stableUserId, + query: input.query, + inboxId: input.inboxId, + direction: input.direction, + processingStatus: input.processingStatus, + deliveryStatus: input.deliveryStatus, + limit: input.limit, + }) + } + + const searched = await mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }).searchMessages({ + query: input.query, + inboxId: input.inboxId, + direction: input.direction, + processingStatus: input.processingStatus, + deliveryStatus: input.deliveryStatus, + classification: input.classification, + limit: input.limit, + }) + return searched.messages.map((message) => + mailboxMessageToEmailMessageRecord(message, input.stableUserId), + ) +} + +/** + * Account inbox page: total + offset page, optional substring query and + * classification filter. Uses Mailbox count/list/search when cutover is on. + */ +export async function listOwnerEmailMessagesPage( + input: OwnerReadBase & { + query: string + classification: EmailClassification | null + pageSize: number + offset: number + }, +): Promise<{ total: number; messages: Array }> { + const useMailbox = await useMailboxReads(input) + if (!useMailbox) { + return listOwnerEmailMessagesPageFromD1(input) + } + + const mailbox = mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }) + const filters = { + classification: input.classification, + query: input.query || null, + } + const [{ total }, listed] = await Promise.all([ + mailbox.countMessages(filters), + input.query + ? mailbox.searchMessages({ + query: input.query, + classification: input.classification, + limit: input.pageSize, + offset: input.offset, + }) + : mailbox.listMessages({ + classification: input.classification, + limit: input.pageSize, + offset: input.offset, + }), + ]) + return { + total, + messages: listed.messages.map((message) => + mailboxMessageToEmailMessageRecord(message, input.stableUserId), + ), + } +} + +async function listOwnerEmailMessagesPageFromD1(input: { + env: MailboxReadCutoverEnv + stableUserId: string + query: string + classification: EmailClassification | null + pageSize: number + offset: number +}): Promise<{ total: number; messages: Array }> { + return listEmailMessagesPageForUser({ + db: input.env.APP_DB, + userId: input.stableUserId, + query: input.query, + classification: input.classification, + pageSize: input.pageSize, + offset: input.offset, + }) +} + +export async function listOwnerEmailAttachmentsForMessage( + input: OwnerReadBase & { messageId: string }, +): Promise> { + const useMailbox = await useMailboxReads(input) + if (!useMailbox) { + return listEmailAttachmentsForMessage({ + db: input.env.APP_DB, + messageId: input.messageId, + }) + } + + const attachments = await mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }).listAttachmentsForMessage({ messageId: input.messageId }) + return attachments.map(mailboxAttachmentToEmailAttachmentRecord) +} + +export async function getOwnerEmailAttachmentRecordById( + input: OwnerReadBase & { attachmentId: string }, +): Promise { + const useMailbox = await useMailboxReads(input) + if (!useMailbox) { + return getEmailAttachmentRecordById({ + db: input.env.APP_DB, + userId: input.stableUserId, + attachmentId: input.attachmentId, + }) + } + + const attachment = await mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }).getAttachment({ attachmentId: input.attachmentId }) + return attachment + ? mailboxAttachmentToEmailAttachmentRecord(attachment) + : null +} + +/** + * Owner-facing attachment fetch: metadata via cutover, bytes via existing + * EMAIL_BLOBS / raw-MIME helpers (never through the DO). + */ +export async function getOwnerEmailAttachmentById( + input: OwnerReadBase & { + blobs: R2Bucket + attachmentId: string + }, +) { + const attachment = await getOwnerEmailAttachmentRecordById(input) + if (!attachment) return null + const message = await getOwnerEmailMessageById({ + ...input, + messageId: attachment.messageId, + }) + return loadEmailAttachmentContent({ + blobs: input.blobs, + attachment, + message, + }) +} + +export async function listOwnerEmailDeliveryEvents( + input: OwnerReadBase & { + messageId?: string | null + eventType?: EmailDeliveryEventType | null + limit: number + }, +): Promise> { + const useMailbox = await useMailboxReads(input) + if (!useMailbox) { + return listEmailDeliveryEvents({ + db: input.env.APP_DB, + userId: input.stableUserId, + messageId: input.messageId, + eventType: input.eventType, + limit: input.limit, + }) + } + + const events = await mailboxRpc({ + env: input.env, + userId: input.stableUserId, + }).listDeliveryEvents({ + messageId: input.messageId, + eventType: input.eventType, + limit: input.limit, + }) + return events.map((event) => + mailboxDeliveryEventToEmailDeliveryEventRecord(event, input.stableUserId), + ) +} + +/** Load raw MIME for an owner message (metadata via cutover, bytes via R2). */ +export async function loadOwnerEmailRawMime( + input: OwnerReadBase & { + blobs: R2Bucket + messageId: string + }, +): Promise { + const message = await getOwnerEmailMessageById(input) + if (!message) return null + return loadRawMime({ blobs: input.blobs, message }) +} + +/** + * Resolve numeric `users.id` for MCP callers that only have a stable id. + * Returns null when missing (callers fail closed to D1 by skipping cutover). + */ +export async function resolveMailboxReadCutoverDbUserId(input: { + db: D1Database + stableUserId: string +}): Promise { + try { + return await resolveDbUserId(input.db, input.stableUserId) + } catch { + return null + } +} diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 75b38f9927..835d9236bb 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -10,10 +10,12 @@ import { decodeMailboxListCursor, encodeMailboxListCursor, mailboxNowIso, + normalizeMailboxOffset, normalizeMailboxPageSize, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, + type MailboxCountMessagesInput, type MailboxCountResult, type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, @@ -26,9 +28,11 @@ import { type MailboxThreadRecord, } from './mailbox-types.ts' import { + type EmailClassification, type EmailDeliveryEventType, type EmailDeliveryStatus, type EmailDirection, + type EmailProcessingStatus, } from './types.ts' import { emailAttachmentBlobKey, emailRawMimeKey } from './blob-keys.ts' import { @@ -57,6 +61,48 @@ export type MailboxUpsertResult = { accepted: boolean } +function buildMailboxMessageFilterClauses(input: { + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + classification?: EmailClassification | null + query?: string | null +}): { clauses: Array; params: Array } { + const clauses: Array = ['1 = 1'] + const params: Array = [] + if (typeof input.query === 'string') { + const pattern = `%${escapeLikePattern(input.query.trim().toLowerCase())}%` + clauses.push(`( + LOWER(subject) LIKE ? ESCAPE '\\' + OR LOWER(from_address) LIKE ? ESCAPE '\\' + OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' + )`) + params.push(pattern, pattern, pattern) + } + if (input.inboxId) { + clauses.push('inbox_id = ?') + params.push(input.inboxId) + } + if (input.direction) { + clauses.push('direction = ?') + params.push(assertMailboxDirection(input.direction)) + } + if (input.processingStatus) { + clauses.push('processing_status = ?') + params.push(assertMailboxProcessingStatus(input.processingStatus)) + } + if (input.deliveryStatus) { + clauses.push('delivery_status = ?') + params.push(assertMailboxDeliveryStatus(input.deliveryStatus)) + } + if (input.classification) { + clauses.push('classification = ?') + params.push(assertMailboxClassification(input.classification)) + } + return { clauses, params } +} + /** * SQLite write/query helpers for one Mailbox DO. No alarm / R2 side effects. */ @@ -504,40 +550,21 @@ export class MailboxStore { nextCursor: string | null } { const limit = normalizeMailboxPageSize(input.limit) - const clauses: Array = ['1 = 1'] - const params: Array = [] - if (input.inboxId) { - clauses.push('inbox_id = ?') - params.push(input.inboxId) - } - if (input.direction) { - clauses.push('direction = ?') - params.push(assertMailboxDirection(input.direction)) - } - if (input.processingStatus) { - clauses.push('processing_status = ?') - params.push(assertMailboxProcessingStatus(input.processingStatus)) - } - if (input.deliveryStatus) { - clauses.push('delivery_status = ?') - params.push(assertMailboxDeliveryStatus(input.deliveryStatus)) - } - if (input.classification) { - clauses.push('classification = ?') - params.push(assertMailboxClassification(input.classification)) - } + const { clauses, params } = buildMailboxMessageFilterClauses(input) if (input.cursor) { const cursor = decodeMailboxListCursor(input.cursor) clauses.push('(created_at, id) < (?, ?)') params.push(cursor.createdAt, cursor.id) } - params.push(limit + 1) + const offset = + input.cursor == null ? normalizeMailboxOffset(input.offset) : 0 + params.push(limit + 1, offset) const rows = this.sql .exec>( `SELECT * FROM email_messages WHERE ${clauses.join(' AND ')} ORDER BY created_at DESC, id DESC - LIMIT ?`, + LIMIT ? OFFSET ?`, ...params, ) .toArray() @@ -563,50 +590,52 @@ export class MailboxStore { throw new Error('Mailbox search query must be a string.') } const limit = normalizeMailboxPageSize(input.limit) - const pattern = `%${escapeLikePattern(input.query.trim().toLowerCase())}%` - const clauses: Array = [ - `( - LOWER(subject) LIKE ? ESCAPE '\\' - OR LOWER(from_address) LIKE ? ESCAPE '\\' - OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' - )`, - ] - const params: Array = [pattern, pattern, pattern] - if (input.inboxId) { - clauses.push('inbox_id = ?') - params.push(input.inboxId) - } - if (input.direction) { - clauses.push('direction = ?') - params.push(assertMailboxDirection(input.direction)) - } - if (input.processingStatus) { - clauses.push('processing_status = ?') - params.push(assertMailboxProcessingStatus(input.processingStatus)) - } - if (input.deliveryStatus) { - clauses.push('delivery_status = ?') - params.push(assertMailboxDeliveryStatus(input.deliveryStatus)) - } - params.push(limit) + const offset = normalizeMailboxOffset(input.offset) + const { clauses, params } = buildMailboxMessageFilterClauses({ + ...input, + query: input.query, + }) + params.push(limit, offset) const rows = this.sql .exec>( `SELECT * FROM email_messages WHERE ${clauses.join(' AND ')} ORDER BY created_at DESC, id DESC - LIMIT ?`, + LIMIT ? OFFSET ?`, ...params, ) .toArray() return { messages: rows.map(mapMailboxMessageRow) } } + countMessages(input: MailboxCountMessagesInput): { total: number } { + const { clauses, params } = buildMailboxMessageFilterClauses(input) + const row = this.sql + .exec<{ n: number }>( + `SELECT COUNT(*) AS n FROM email_messages + WHERE ${clauses.join(' AND ')}`, + ...params, + ) + .one() + return { total: Number(row.n ?? 0) || 0 } + } + + getAttachment(attachmentId: string): MailboxAttachmentRecord | null { + const row = this.sql + .exec>( + `SELECT * FROM email_attachments WHERE id = ? LIMIT 1`, + assertMailboxNonEmptyString(attachmentId, 'attachmentId'), + ) + .toArray()[0] + return row ? mapMailboxAttachmentRow(row) : null + } + listAttachmentsForMessage(messageId: string): Array { return this.sql .exec>( `SELECT * FROM email_attachments WHERE message_id = ? - ORDER BY id ASC`, + ORDER BY created_at ASC, id ASC`, assertMailboxNonEmptyString(messageId, 'messageId'), ) .toArray() diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 2cbabe3466..1549310751 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -343,7 +343,10 @@ export type MailboxListMessagesInput = { deliveryStatus?: EmailDeliveryStatus | null classification?: EmailClassification | null limit?: number + /** Keyset cursor; when set, `offset` is ignored. */ cursor?: string | null + /** Offset page (account inbox). Ignored when `cursor` is set. */ + offset?: number | null } export type MailboxSearchMessagesInput = { @@ -352,7 +355,19 @@ export type MailboxSearchMessagesInput = { direction?: EmailDirection | null processingStatus?: EmailProcessingStatus | null deliveryStatus?: EmailDeliveryStatus | null + classification?: EmailClassification | null limit?: number + offset?: number | null +} + +export type MailboxCountMessagesInput = { + inboxId?: string | null + direction?: EmailDirection | null + processingStatus?: EmailProcessingStatus | null + deliveryStatus?: EmailDeliveryStatus | null + classification?: EmailClassification | null + /** When set, applies the same subject/from/envelope substring match as search. */ + query?: string | null } export type MailboxListCursorPayload = { @@ -533,6 +548,12 @@ export type MailboxRpc = { searchMessages: (input: MailboxSearchMessagesInput) => Promise<{ messages: Array }> + countMessages: ( + input: MailboxCountMessagesInput, + ) => Promise<{ total: number }> + getAttachment: (input: { + attachmentId: string + }) => Promise listAttachmentsForMessage: (input: { messageId: string }) => Promise> @@ -572,6 +593,12 @@ export function normalizeMailboxPageSize(pageSize: number | undefined) { return Math.min(Math.max(requested, 1), mailboxMaxPageSize) } +/** Non-negative offset; invalid/omitted values become 0. */ +export function normalizeMailboxOffset(offset: number | null | undefined) { + if (typeof offset !== 'number' || !Number.isFinite(offset)) return 0 + return Math.max(0, Math.trunc(offset)) +} + export function assertMailboxNonEmptyString( value: unknown, label: string, diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 754c50504c..7027dd55f5 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -970,6 +970,109 @@ export async function listEmailMessages(input: { return (result.results ?? []).map(mapMessageRow) } +/** + * Account-inbox page query: total + OFFSET page with optional substring search + * and classification filter. Used by the cutover D1 off-path. + */ +export async function listEmailMessagesPageForUser(input: { + db: D1Database + userId: string + query: string + classification: EmailClassification | null + pageSize: number + offset: number +}): Promise<{ total: number; messages: Array }> { + const classification = input.classification + if (input.query) { + const pattern = `%${escapeLikePattern(input.query.toLowerCase())}%` + const [totalResult, messageRows] = await Promise.all([ + input.db + .prepare( + `SELECT COUNT(*) AS total + FROM email_messages + WHERE user_id = ? + AND (? IS NULL OR classification = ?) + AND ( + LOWER(subject) LIKE ? ESCAPE '\\' + OR LOWER(from_address) LIKE ? ESCAPE '\\' + OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' + )`, + ) + .bind( + input.userId, + classification, + classification, + pattern, + pattern, + pattern, + ) + .first<{ total: number }>(), + input.db + .prepare( + `SELECT * + FROM email_messages + WHERE user_id = ? + AND (? IS NULL OR classification = ?) + AND ( + LOWER(subject) LIKE ? ESCAPE '\\' + OR LOWER(from_address) LIKE ? ESCAPE '\\' + OR LOWER(COALESCE(envelope_from, '')) LIKE ? ESCAPE '\\' + ) + ORDER BY created_at DESC, id DESC + LIMIT ? OFFSET ?`, + ) + .bind( + input.userId, + classification, + classification, + pattern, + pattern, + pattern, + input.pageSize, + input.offset, + ) + .all>(), + ]) + return { + total: Number(totalResult?.total ?? 0), + messages: (messageRows.results ?? []).map(mapMessageRow), + } + } + + const [totalResult, messageRows] = await Promise.all([ + input.db + .prepare( + `SELECT COUNT(*) AS total + FROM email_messages + WHERE user_id = ? + AND (? IS NULL OR classification = ?)`, + ) + .bind(input.userId, classification, classification) + .first<{ total: number }>(), + input.db + .prepare( + `SELECT * + FROM email_messages + WHERE user_id = ? + AND (? IS NULL OR classification = ?) + ORDER BY created_at DESC, id DESC + LIMIT ? OFFSET ?`, + ) + .bind( + input.userId, + classification, + classification, + input.pageSize, + input.offset, + ) + .all>(), + ]) + return { + total: Number(totalResult?.total ?? 0), + messages: (messageRows.results ?? []).map(mapMessageRow), + } +} + export async function listEmailDeliveryEvents(input: { db: D1Database userId: string diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index 2dc1d41770..646320339b 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -34,6 +34,7 @@ import { } from './repo.ts' import { evaluateEmailSenderRules } from './sender-rules.ts' import { + type EmailAttachmentRecord, type EmailClassification, type EmailDeliveryEventRecord, type EmailDeliveryStatus, @@ -541,19 +542,17 @@ export async function storeIdempotentInboundEmail(input: { } } -export async function getEmailAttachmentById(input: { - db: D1Database - /** EMAIL_BLOBS bucket for messages whose raw MIME lives in R2. */ +/** + * Load attachment bytes after metadata selection. External attachments use + * `EMAIL_BLOBS` via `storageKey`; raw-mime attachments parse the message's + * raw MIME blob. Metadata may come from D1 or Mailbox. + */ +export async function loadEmailAttachmentContent(input: { blobs: R2Bucket - userId: string - attachmentId: string + attachment: EmailAttachmentRecord + message: Pick | null }) { - const attachment = await getEmailAttachmentRecordById({ - db: input.db, - userId: input.userId, - attachmentId: input.attachmentId, - }) - if (!attachment) return null + const { attachment } = input // Externally stored attachments (outbound mail) keep their bytes in a // dedicated R2 object instead of inside raw MIME. if (attachment.storageKind === 'external') { @@ -573,13 +572,8 @@ export async function getEmailAttachmentById(input: { if (attachment.storageKind === 'unavailable') { return { ...attachment, content: null, contentBase64: null } } - const message = await getEmailMessageById({ - db: input.db, - userId: input.userId, - messageId: attachment.messageId, - }) - const rawMime = message - ? await loadRawMime({ blobs: input.blobs, message }) + const rawMime = input.message + ? await loadRawMime({ blobs: input.blobs, message: input.message }) : null if (!rawMime) { return { @@ -623,6 +617,31 @@ export async function getEmailAttachmentById(input: { } } +export async function getEmailAttachmentById(input: { + db: D1Database + /** EMAIL_BLOBS bucket for messages whose raw MIME lives in R2. */ + blobs: R2Bucket + userId: string + attachmentId: string +}) { + const attachment = await getEmailAttachmentRecordById({ + db: input.db, + userId: input.userId, + attachmentId: input.attachmentId, + }) + if (!attachment) return null + const message = await getEmailMessageById({ + db: input.db, + userId: input.userId, + messageId: attachment.messageId, + }) + return loadEmailAttachmentContent({ + blobs: input.blobs, + attachment, + message, + }) +} + /** * How many quota/size/verification rejections per inbox per UTC day get * their own delivery-event row before collapsing into the daily aggregate diff --git a/packages/worker/src/feature-flags/registry.ts b/packages/worker/src/feature-flags/registry.ts index 9900341d0b..c344cd9d09 100644 --- a/packages/worker/src/feature-flags/registry.ts +++ b/packages/worker/src/feature-flags/registry.ts @@ -53,7 +53,7 @@ export const featureFlagDefinitions = [ key: 'mailbox-read-cutover', defaultEnabled: false, description: - 'Serve owner-scoped user-mail message reads from the per-user Mailbox Durable Object when the account also has continuous D1↔Mailbox parity soak (matching_since ≥ 24h, fresh checked_at, zero mismatches). Default off; fail closed without verified parity. No live reader is switched until a later wiring PR.', + 'Serve owner-facing user-mail reads (app inbox + MCP list/get/search/attachment/delivery-events) from the per-user Mailbox Durable Object when the account also has continuous D1↔Mailbox parity soak (matching_since ≥ 2h pre-launch, fresh checked_at, zero mismatches). Default off; deploy first, then enable per user. Fail closed without verified parity.', successMetric: { eventType: 'email_received', measure: 'error_rate', diff --git a/packages/worker/src/feature-flags/service.ts b/packages/worker/src/feature-flags/service.ts index 7e6604f0b2..d24521403f 100644 --- a/packages/worker/src/feature-flags/service.ts +++ b/packages/worker/src/feature-flags/service.ts @@ -112,11 +112,11 @@ function normalizeFeatureFlagNote(note: unknown): string | null { return trimmed } -export async function isFeatureEnabled( +export async function evaluateFeatureFlag( db: D1Database, key: FeatureFlagKey, userId: number | null, -): Promise { +): Promise { if (userId !== null) { const override = await db .prepare( @@ -127,7 +127,10 @@ export async function isFeatureEnabled( .bind(key, userId) .first<{ enabled: number }>() if (override) { - return override.enabled === 1 + return { + enabled: override.enabled === 1, + source: 'override', + } } } @@ -151,7 +154,15 @@ export async function isFeatureEnabled( } : null, defaultEnabled: getFeatureFlagDefinition(key).defaultEnabled, - }).enabled + }) +} + +export async function isFeatureEnabled( + db: D1Database, + key: FeatureFlagKey, + userId: number | null, +): Promise { + return (await evaluateFeatureFlag(db, key, userId)).enabled } export async function getFeatureFlagEvaluationsForUser( diff --git a/packages/worker/src/mcp/capabilities/email/email-attachment-get.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-attachment-get.node.test.ts index ae9ba291d4..e882a65582 100644 --- a/packages/worker/src/mcp/capabilities/email/email-attachment-get.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-attachment-get.node.test.ts @@ -1,12 +1,15 @@ import { expect, test, vi } from 'vitest' const mockModule = vi.hoisted(() => ({ - getEmailAttachmentById: vi.fn(), + getOwnerEmailAttachmentById: vi.fn(), + resolveMailboxReadCutoverDbUserId: vi.fn(), })) -vi.mock('#worker/email/service.ts', () => ({ - getEmailAttachmentById: (...args: Array) => - mockModule.getEmailAttachmentById(...args), +vi.mock('#worker/email/mailbox-read-cutover.ts', () => ({ + getOwnerEmailAttachmentById: (...args: Array) => + mockModule.getOwnerEmailAttachmentById(...args), + resolveMailboxReadCutoverDbUserId: (...args: Array) => + mockModule.resolveMailboxReadCutoverDbUserId(...args), })) const { emailAttachmentGetCapability } = @@ -45,7 +48,10 @@ test('email capabilities require a signed-in user and return attachment content }, }) const verifiedDb = createUsersDb('2026-01-01T00:00:00.000Z') - const env = { APP_DB: verifiedDb } as Env + const env = { + APP_DB: verifiedDb, + EMAIL_BLOBS: {} as R2Bucket, + } as Env await expect( emailAttachmentGetCapability.handler( @@ -56,9 +62,10 @@ test('email capabilities require a signed-in user and return attachment content }, ), ).rejects.toThrow(/Account email is not verified/) - expect(mockModule.getEmailAttachmentById).not.toHaveBeenCalled() + expect(mockModule.getOwnerEmailAttachmentById).not.toHaveBeenCalled() - mockModule.getEmailAttachmentById.mockResolvedValueOnce({ + mockModule.resolveMailboxReadCutoverDbUserId.mockResolvedValueOnce(7) + mockModule.getOwnerEmailAttachmentById.mockResolvedValueOnce({ id: 'attachment-1', messageId: 'message-1', filename: 'note.txt', @@ -77,9 +84,11 @@ test('email capabilities require a signed-in user and return attachment content { env, callerContext }, ) - expect(mockModule.getEmailAttachmentById).toHaveBeenCalledWith({ - db: verifiedDb, - userId: 'user-1', + expect(mockModule.getOwnerEmailAttachmentById).toHaveBeenCalledWith({ + env, + dbUserId: 7, + stableUserId: 'user-1', + blobs: env.EMAIL_BLOBS, attachmentId: 'attachment-1', }) expect(result).toEqual({ @@ -93,7 +102,8 @@ test('email capabilities require a signed-in user and return attachment content data_base64: 'QXR0YWNobWVudCB0ZXh0', }) - mockModule.getEmailAttachmentById.mockResolvedValueOnce(null) + mockModule.resolveMailboxReadCutoverDbUserId.mockResolvedValueOnce(7) + mockModule.getOwnerEmailAttachmentById.mockResolvedValueOnce(null) await expect( emailAttachmentGetCapability.handler( diff --git a/packages/worker/src/mcp/capabilities/email/email-attachment-get.ts b/packages/worker/src/mcp/capabilities/email/email-attachment-get.ts index 82681d764d..66f0f4d8de 100644 --- a/packages/worker/src/mcp/capabilities/email/email-attachment-get.ts +++ b/packages/worker/src/mcp/capabilities/email/email-attachment-get.ts @@ -1,8 +1,11 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' +import { + getOwnerEmailAttachmentById, + resolveMailboxReadCutoverDbUserId, +} from '#worker/email/mailbox-read-cutover.ts' import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' -import { getEmailAttachmentById } from '#worker/email/service.ts' const emailAttachmentContentSchema = z.object({ id: z.string(), @@ -31,10 +34,18 @@ export const emailAttachmentGetCapability = defineDomainCapability( outputSchema: emailAttachmentContentSchema, async handler(args, ctx) { const user = await requireVerifiedEmailAccountUser(ctx) - const attachment = await getEmailAttachmentById({ + const dbUserId = await resolveMailboxReadCutoverDbUserId({ db: ctx.env.APP_DB, + stableUserId: user.userId, + }) + if (dbUserId === null) { + throw new Error('Authenticated user could not be resolved.') + } + const attachment = await getOwnerEmailAttachmentById({ + env: ctx.env, + dbUserId, + stableUserId: user.userId, blobs: ctx.env.EMAIL_BLOBS, - userId: user.userId, attachmentId: args.attachment_id, }) if (!attachment) { diff --git a/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.node.test.ts index 23337423b3..dc8ca52464 100644 --- a/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.node.test.ts @@ -2,11 +2,13 @@ import { expect, test, vi } from 'vitest' import { createMcpCallerContext } from '#mcp/context.ts' const mocks = vi.hoisted(() => ({ - listEmailDeliveryEvents: vi.fn(), + listOwnerEmailDeliveryEvents: vi.fn(), + resolveMailboxReadCutoverDbUserId: vi.fn(), })) -vi.mock('#worker/email/repo.ts', () => ({ - listEmailDeliveryEvents: mocks.listEmailDeliveryEvents, +vi.mock('#worker/email/mailbox-read-cutover.ts', () => ({ + listOwnerEmailDeliveryEvents: mocks.listOwnerEmailDeliveryEvents, + resolveMailboxReadCutoverDbUserId: mocks.resolveMailboxReadCutoverDbUserId, })) const { emailDeliveryEventListCapability } = @@ -37,7 +39,8 @@ function createContext() { } test('email_delivery_event_list returns parsed events scoped to the signed-in user', async () => { - mocks.listEmailDeliveryEvents.mockResolvedValueOnce([ + mocks.resolveMailboxReadCutoverDbUserId.mockResolvedValueOnce(7) + mocks.listOwnerEmailDeliveryEvents.mockResolvedValueOnce([ { id: 'event-1', messageId: 'message-1', @@ -60,9 +63,10 @@ test('email_delivery_event_list returns parsed events scoped to the signed-in us context, ) - expect(mocks.listEmailDeliveryEvents).toHaveBeenCalledWith({ - db: context.env.APP_DB, - userId: 'user-1', + expect(mocks.listOwnerEmailDeliveryEvents).toHaveBeenCalledWith({ + env: context.env, + dbUserId: 7, + stableUserId: 'user-1', messageId: 'message-1', eventType: 'bounced', limit: 10, diff --git a/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.ts b/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.ts index e5e026ca11..4dc22b8422 100644 --- a/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.ts +++ b/packages/worker/src/mcp/capabilities/email/email-delivery-event-list.ts @@ -1,7 +1,10 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' -import { listEmailDeliveryEvents } from '#worker/email/repo.ts' +import { + listOwnerEmailDeliveryEvents, + resolveMailboxReadCutoverDbUserId, +} from '#worker/email/mailbox-read-cutover.ts' import { emailDeliveryEventTypeValues } from '#worker/email/types.ts' import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' @@ -57,9 +60,17 @@ export const emailDeliveryEventListCapability = defineDomainCapability( }), async handler(args, ctx) { const user = await requireVerifiedEmailAccountUser(ctx) - const events = await listEmailDeliveryEvents({ + const dbUserId = await resolveMailboxReadCutoverDbUserId({ db: ctx.env.APP_DB, - userId: user.userId, + stableUserId: user.userId, + }) + if (dbUserId === null) { + throw new Error('Authenticated user could not be resolved.') + } + const events = await listOwnerEmailDeliveryEvents({ + env: ctx.env, + dbUserId, + stableUserId: user.userId, messageId: args.message_id ?? null, eventType: args.event_type ?? null, limit: args.limit, diff --git a/packages/worker/src/mcp/capabilities/email/email-message-get.ts b/packages/worker/src/mcp/capabilities/email/email-message-get.ts index a7a222fd30..2aff53cca6 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-get.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-get.ts @@ -1,11 +1,13 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' -import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' import { - getEmailMessageById, - listEmailAttachmentsForMessage, -} from '#worker/email/repo.ts' + getOwnerEmailMessageById, + listOwnerEmailAttachmentsForMessage, + resolveMailboxReadCutoverDbUserId, + type MailboxReadCutoverMemo, +} from '#worker/email/mailbox-read-cutover.ts' +import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' import { emailMessageDetailSchema, toMessageDetail } from './shared.ts' export const emailMessageGetCapability = defineDomainCapability( @@ -24,17 +26,30 @@ export const emailMessageGetCapability = defineDomainCapability( outputSchema: emailMessageDetailSchema, async handler(args, ctx) { const user = await requireVerifiedEmailAccountUser(ctx) - const message = await getEmailMessageById({ + const dbUserId = await resolveMailboxReadCutoverDbUserId({ db: ctx.env.APP_DB, - userId: user.userId, + stableUserId: user.userId, + }) + if (dbUserId === null) { + throw new Error('Authenticated user could not be resolved.') + } + const memo: MailboxReadCutoverMemo = {} + const message = await getOwnerEmailMessageById({ + env: ctx.env, + dbUserId, + stableUserId: user.userId, messageId: args.message_id, + memo, }) if (!message) { throw new Error(`Email message not found: ${args.message_id}`) } - const attachments = await listEmailAttachmentsForMessage({ - db: ctx.env.APP_DB, + const attachments = await listOwnerEmailAttachmentsForMessage({ + env: ctx.env, + dbUserId, + stableUserId: user.userId, messageId: message.id, + memo, }) return toMessageDetail(message, attachments) }, diff --git a/packages/worker/src/mcp/capabilities/email/email-message-list.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-message-list.node.test.ts index c2988ac5a9..914521aea0 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-list.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-list.node.test.ts @@ -2,11 +2,13 @@ import { expect, test, vi } from 'vitest' import { createMcpCallerContext } from '#mcp/context.ts' const mocks = vi.hoisted(() => ({ - listEmailMessages: vi.fn(), + listOwnerEmailMessages: vi.fn(), + resolveMailboxReadCutoverDbUserId: vi.fn(), })) -vi.mock('#worker/email/repo.ts', () => ({ - listEmailMessages: mocks.listEmailMessages, +vi.mock('#worker/email/mailbox-read-cutover.ts', () => ({ + listOwnerEmailMessages: mocks.listOwnerEmailMessages, + resolveMailboxReadCutoverDbUserId: mocks.resolveMailboxReadCutoverDbUserId, })) const { emailMessageListCapability } = await import('./email-message-list.ts') @@ -37,7 +39,8 @@ function createContext() { test('email_message_list forwards classification filter and returns classification fields', async () => { const context = createContext() - mocks.listEmailMessages.mockResolvedValueOnce([ + mocks.resolveMailboxReadCutoverDbUserId.mockResolvedValueOnce(7) + mocks.listOwnerEmailMessages.mockResolvedValueOnce([ { id: 'message-1', direction: 'inbound', @@ -67,9 +70,10 @@ test('email_message_list forwards classification filter and returns classificati context, ) - expect(mocks.listEmailMessages).toHaveBeenCalledWith({ - db: context.env.APP_DB, - userId: 'user-1', + expect(mocks.listOwnerEmailMessages).toHaveBeenCalledWith({ + env: context.env, + dbUserId: 7, + stableUserId: 'user-1', inboxId: null, direction: null, processingStatus: null, diff --git a/packages/worker/src/mcp/capabilities/email/email-message-list.ts b/packages/worker/src/mcp/capabilities/email/email-message-list.ts index fb3471cc44..9e104588e5 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-list.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-list.ts @@ -1,12 +1,15 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' -import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' -import { listEmailMessages } from '#worker/email/repo.ts' +import { + listOwnerEmailMessages, + resolveMailboxReadCutoverDbUserId, +} from '#worker/email/mailbox-read-cutover.ts' import { emailClassificationValues, emailDeliveryStatusValues, } from '#worker/email/types.ts' +import { requireVerifiedEmailAccountUser } from './require-verified-user.ts' import { emailMessageSummarySchema, toMessageSummary } from './shared.ts' export const emailMessageListCapability = defineDomainCapability( @@ -32,9 +35,17 @@ export const emailMessageListCapability = defineDomainCapability( }), async handler(args, ctx) { const user = await requireVerifiedEmailAccountUser(ctx) - const messages = await listEmailMessages({ + const dbUserId = await resolveMailboxReadCutoverDbUserId({ db: ctx.env.APP_DB, - userId: user.userId, + stableUserId: user.userId, + }) + if (dbUserId === null) { + throw new Error('Authenticated user could not be resolved.') + } + const messages = await listOwnerEmailMessages({ + env: ctx.env, + dbUserId, + stableUserId: user.userId, inboxId: args.inbox_id ?? null, direction: args.direction ?? null, processingStatus: args.processing_status ?? null, diff --git a/packages/worker/src/mcp/capabilities/email/email-message-search.node.test.ts b/packages/worker/src/mcp/capabilities/email/email-message-search.node.test.ts index 279f0f937c..06bdf37eff 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-search.node.test.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-search.node.test.ts @@ -1,12 +1,15 @@ import { expect, test, vi } from 'vitest' const mockModule = vi.hoisted(() => ({ - searchEmailMessages: vi.fn(), + searchOwnerEmailMessages: vi.fn(), + resolveMailboxReadCutoverDbUserId: vi.fn(), })) -vi.mock('#worker/email/repo.ts', () => ({ - searchEmailMessages: (...args: Array) => - mockModule.searchEmailMessages(...args), +vi.mock('#worker/email/mailbox-read-cutover.ts', () => ({ + searchOwnerEmailMessages: (...args: Array) => + mockModule.searchOwnerEmailMessages(...args), + resolveMailboxReadCutoverDbUserId: (...args: Array) => + mockModule.resolveMailboxReadCutoverDbUserId(...args), })) const { emailMessageSearchCapability } = @@ -66,10 +69,11 @@ test('email_message_search requires a signed-in, verified user and forwards the }, ), ).rejects.toThrow(/Account email is not verified/) - expect(mockModule.searchEmailMessages).not.toHaveBeenCalled() + expect(mockModule.searchOwnerEmailMessages).not.toHaveBeenCalled() const env = createEnv() - mockModule.searchEmailMessages.mockResolvedValueOnce([ + mockModule.resolveMailboxReadCutoverDbUserId.mockResolvedValueOnce(7) + mockModule.searchOwnerEmailMessages.mockResolvedValueOnce([ { id: 'message-1', direction: 'inbound', @@ -102,9 +106,10 @@ test('email_message_search requires a signed-in, verified user and forwards the { env, callerContext: createUserContext() }, ) - expect(mockModule.searchEmailMessages).toHaveBeenCalledWith({ - db: env.APP_DB, - userId: 'user-1', + expect(mockModule.searchOwnerEmailMessages).toHaveBeenCalledWith({ + env, + dbUserId: 7, + stableUserId: 'user-1', query: 'invoice', inboxId: 'inbox-1', direction: 'inbound', diff --git a/packages/worker/src/mcp/capabilities/email/email-message-search.ts b/packages/worker/src/mcp/capabilities/email/email-message-search.ts index b55f948bb7..39d0c0f2a0 100644 --- a/packages/worker/src/mcp/capabilities/email/email-message-search.ts +++ b/packages/worker/src/mcp/capabilities/email/email-message-search.ts @@ -1,7 +1,10 @@ import { z } from 'zod' import { defineDomainCapability } from '#mcp/capabilities/define-domain-capability.ts' import { capabilityDomainNames } from '#mcp/capabilities/domain-metadata.ts' -import { searchEmailMessages } from '#worker/email/repo.ts' +import { + resolveMailboxReadCutoverDbUserId, + searchOwnerEmailMessages, +} from '#worker/email/mailbox-read-cutover.ts' import { emailDeliveryStatusValues, emailDirectionValues, @@ -40,9 +43,17 @@ export const emailMessageSearchCapability = defineDomainCapability( }), async handler(args, ctx) { const user = await requireVerifiedEmailAccountUser(ctx) - const messages = await searchEmailMessages({ + const dbUserId = await resolveMailboxReadCutoverDbUserId({ db: ctx.env.APP_DB, - userId: user.userId, + stableUserId: user.userId, + }) + if (dbUserId === null) { + throw new Error('Authenticated user could not be resolved.') + } + const messages = await searchOwnerEmailMessages({ + env: ctx.env, + dbUserId, + stableUserId: user.userId, query: args.query, inboxId: args.inbox_id ?? null, direction: args.direction ?? null, From ff05007bc2f9fa5a292eeb747e1769cdb34b8d1f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 02:44:55 +0000 Subject: [PATCH 41/62] fix(email): scope cutover attachments and exposures Co-authored-by: Kent C. Dodds --- .../src/app/account-email-data.node.test.ts | 198 ++++++++++++++++++ packages/worker/src/app/account-email-data.ts | 10 +- .../account-email-exposure.node.test.ts | 106 ++++++++++ .../worker/src/app/handlers/account-email.ts | 1 + .../email/mailbox-read-cutover.node.test.ts | 63 +++++- .../worker/src/email/mailbox-read-cutover.ts | 5 +- .../email/repo-attachments.workers.test.ts | 60 ++++++ packages/worker/src/email/repo.ts | 19 ++ 8 files changed, 455 insertions(+), 7 deletions(-) create mode 100644 packages/worker/src/app/account-email-data.node.test.ts create mode 100644 packages/worker/src/app/handlers/account-email-exposure.node.test.ts create mode 100644 packages/worker/src/email/repo-attachments.workers.test.ts diff --git a/packages/worker/src/app/account-email-data.node.test.ts b/packages/worker/src/app/account-email-data.node.test.ts new file mode 100644 index 0000000000..050a27d715 --- /dev/null +++ b/packages/worker/src/app/account-email-data.node.test.ts @@ -0,0 +1,198 @@ +import { expect, test, vi } from 'vitest' +import type * as EntitlementPlans from '#worker/entitlements/plans.ts' +import type * as EntitlementService from '#worker/entitlements/service.ts' +import { createInMemoryUserMeterEnv } from '#worker/test-support/user-meter.ts' +import { mailboxReadCutoverFlagKey } from '#worker/email/mailbox-read-cutover.ts' + +const messageRecord = { + id: 'msg-1', + direction: 'inbound' as const, + userId: 'stable-user-1', + inboxId: 'inbox-1', + threadId: null, + senderIdentityId: null, + fromAddress: 'sender@example.com', + envelopeFrom: 'sender@example.com', + toAddresses: ['user@inbox.example.com'], + ccAddresses: [], + bccAddresses: [], + replyToAddresses: [], + subject: 'Hello from sender', + messageIdHeader: '', + inReplyToHeader: null, + references: [], + headers: { subject: ['Hello from sender'] }, + authResults: null, + textBody: 'Plain text body', + htmlBody: '

HTML body

', + rawMimeKey: null, + rawSize: 128, + processingStatus: 'stored' as const, + classification: 'accepted' as const, + classificationReason: null, + providerMessageId: null, + deliveryStatus: null, + deliveryStatusAt: null, + error: null, + receivedAt: new Date(0).toISOString(), + sentAt: null, + createdAt: new Date(0).toISOString(), + updatedAt: new Date(0).toISOString(), +} + +const { recordFeatureFlagExposuresMock } = vi.hoisted(() => ({ + recordFeatureFlagExposuresMock: vi.fn(async () => {}), +})) + +vi.mock('#app/email-verification.ts', () => ({ + emailVerificationRequiredMessage: 'verify', + isAccountEmailVerified: vi.fn(async () => true), +})) + +vi.mock('#worker/entitlements/service.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + getUserPlan: vi.fn(async () => 'free' as const), + readEntitlementResourceUsage: vi.fn(async () => 0), + readDailyEntitlementResourceUsage: vi.fn(async () => 0), + } +}) + +vi.mock('#worker/entitlements/plans.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + resolveEmailResourceLimit: vi.fn(() => 100), + } +}) + +vi.mock('#worker/email/platform-address.ts', () => ({ + getPlatformEmailDomain: vi.fn(() => null), + buildPlatformEmailAddress: vi.fn(() => null), +})) + +vi.mock('#worker/email/repo.ts', () => ({ + listEmailInboxesForUser: vi.fn(async () => []), + listEmailInboxAddressesForUser: vi.fn(async () => []), + listEmailMessagesPageForUser: vi.fn(async () => ({ + total: 1, + messages: [messageRecord], + })), + getEmailMessageById: vi.fn(async () => messageRecord), + listEmailAttachmentsForUserMessage: vi.fn(async () => []), + listEmailDeliveryEvents: vi.fn(async () => []), +})) + +vi.mock('#worker/feature-flags/exposure.ts', () => ({ + recordFeatureFlagExposures: (...args: Array) => + recordFeatureFlagExposuresMock(...args), +})) + +const { loadAccountEmailData } = await import('./account-email-data.ts') + +const authenticatedUser = { + sessionUserId: '42', + userId: 42, + username: 'test-user', + email: 'user@example.com', + emailVerified: true, + displayName: 'user', + roles: ['user'] as const, + permissions: [] as const, + artifactOwnerIds: [], + mcpUser: { + userId: 'stable-user-1', + email: 'user@example.com', + username: 'test-user', + displayName: 'user', + }, +} + +function createEnv() { + const meter = createInMemoryUserMeterEnv() + return { + APP_DB: createCutoverDb(true), + APP_BASE_URL: 'https://example.com', + COOKIE_SECRET: 'secret', + ...meter.env, + } as Env +} + +function createCutoverDb(enabled: boolean) { + return { + prepare(query: string) { + const normalized = query.replace(/\s+/g, ' ').trim().toLowerCase() + return { + bind(...params: Array) { + return { + async first() { + if ( + normalized.includes('from feature_flag_user_overrides') && + normalized.includes('where flag_key = ? and user_id = ?') + ) { + expect(params[0]).toBe(mailboxReadCutoverFlagKey) + return { enabled: enabled ? 1 : 0 } as T + } + if ( + normalized.includes('from feature_flags') && + normalized.includes('where key = ?') + ) { + return null + } + if ( + normalized.includes('from users') && + normalized.includes('mailbox_parity_matching_since') + ) { + return null + } + throw new Error(`Unsupported first query: ${query}`) + }, + } + }, + } + }, + } as unknown as D1Database +} + +test('loadAccountEmailData skips cutover exposure when SSR requests it', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(new Date('2026-07-31T15:00:00.000Z')) + recordFeatureFlagExposuresMock.mockClear() + + await loadAccountEmailData({ + env: createEnv(), + request: new Request('https://example.com/account/email?selected=msg-1'), + user: authenticatedUser, + skipCutoverExposureRecording: true, + }) + + expect(recordFeatureFlagExposuresMock).not.toHaveBeenCalled() + vi.useRealTimers() +}) + +test('loadAccountEmailData records cutover exposure once for JSON API loads', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(new Date('2026-07-31T15:00:00.000Z')) + recordFeatureFlagExposuresMock.mockClear() + + await loadAccountEmailData({ + env: createEnv(), + request: new Request( + 'https://example.com/account/email.json?selected=msg-1', + ), + user: authenticatedUser, + }) + + expect(recordFeatureFlagExposuresMock).toHaveBeenCalledTimes(1) + expect(recordFeatureFlagExposuresMock).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + stableUserId: 'stable-user-1', + evaluations: expect.objectContaining({ + [mailboxReadCutoverFlagKey]: expect.any(Object), + }), + }), + ) + vi.useRealTimers() +}) diff --git a/packages/worker/src/app/account-email-data.ts b/packages/worker/src/app/account-email-data.ts index 5ace471fc4..5f6efbe700 100644 --- a/packages/worker/src/app/account-email-data.ts +++ b/packages/worker/src/app/account-email-data.ts @@ -408,6 +408,12 @@ export async function loadAccountEmailData(input: { request: Request user: AuthenticatedUser pathMessageId?: string + /** + * SSR account pages resolve feature flags (and record exposures) in + * `renderAppPage` / session load. Skip cutover exposure here to avoid + * double-recording on the same request. + */ + skipCutoverExposureRecording?: boolean }): Promise { const url = new URL(input.request.url, 'http://localhost') const { page, pageSize, offset } = readPagination(url, { @@ -443,7 +449,9 @@ export async function loadAccountEmailData(input: { ) const cutoverMemo: MailboxReadCutoverMemo = {} const dbUserId = input.user.userId - const skipExposureRecording = hasResolvedRequestFeatureFlags(input.request) + const skipExposureRecording = + input.skipCutoverExposureRecording === true || + hasResolvedRequestFeatureFlags(input.request) const [listResult, inboxes, usage, selectedMessage] = await Promise.all([ listOwnerEmailMessagesPage({ diff --git a/packages/worker/src/app/handlers/account-email-exposure.node.test.ts b/packages/worker/src/app/handlers/account-email-exposure.node.test.ts new file mode 100644 index 0000000000..0588dad751 --- /dev/null +++ b/packages/worker/src/app/handlers/account-email-exposure.node.test.ts @@ -0,0 +1,106 @@ +import { expect, test, vi } from 'vitest' + +const loadAccountEmailDataMock = vi.hoisted(() => + vi.fn(async () => ({ + ok: true as const, + emailVerified: true, + email: 'user@example.com', + username: 'test-user', + inboxAddress: null, + verificationMessage: null, + inboxes: [], + messages: [], + selectedMessage: null, + usage: null, + page: 1, + pageSize: 25, + total: 0, + query: '', + classification: null, + })), +) + +vi.mock('#app/account-email-data.ts', async (importOriginal) => { + const actual = + await importOriginal() + return { + ...actual, + loadAccountEmailData: (...args: Array) => + loadAccountEmailDataMock(...args), + } +}) + +vi.mock('#app/page-auth.ts', () => ({ + requireAuthenticatedPageUser: vi.fn(async () => ({ + sessionUserId: '42', + userId: 42, + username: 'test-user', + email: 'user@example.com', + emailVerified: true, + displayName: 'user', + roles: ['user'], + permissions: [], + artifactOwnerIds: [], + mcpUser: { + userId: 'stable-user-1', + email: 'user@example.com', + username: 'test-user', + displayName: 'user', + }, + })), +})) + +vi.mock('#app/authenticated-user.ts', () => ({ + readAuthenticatedAppUser: vi.fn(async () => ({ + sessionUserId: '42', + userId: 42, + username: 'test-user', + email: 'user@example.com', + emailVerified: true, + displayName: 'user', + roles: ['user'], + permissions: [], + artifactOwnerIds: [], + mcpUser: { + userId: 'stable-user-1', + email: 'user@example.com', + username: 'test-user', + displayName: 'user', + }, + })), +})) + +vi.mock('#app/ssr-render.tsx', () => ({ + renderAppPage: vi.fn(async () => new Response('ok')), +})) + +const { createAccountEmailHandler, createAccountEmailApiHandler } = + await import('./account-email.ts') + +test('SSR account email handler skips cutover exposure recording', async () => { + loadAccountEmailDataMock.mockClear() + const request = new Request('https://example.com/account/email') + const handler = createAccountEmailHandler({} as Env) + await handler.handler({ request, params: {} }) + expect(loadAccountEmailDataMock).toHaveBeenCalledWith( + expect.objectContaining({ + request, + skipCutoverExposureRecording: true, + }), + ) +}) + +test('JSON account email API does not skip cutover exposure recording', async () => { + loadAccountEmailDataMock.mockClear() + const request = new Request('https://example.com/account/email.json') + const handler = createAccountEmailApiHandler({} as Env) + await handler.handler({ request }) + expect(loadAccountEmailDataMock).toHaveBeenCalledWith( + expect.objectContaining({ + request, + }), + ) + expect(loadAccountEmailDataMock.mock.calls[0]?.[0]).not.toHaveProperty( + 'skipCutoverExposureRecording', + ) +}) diff --git a/packages/worker/src/app/handlers/account-email.ts b/packages/worker/src/app/handlers/account-email.ts index e1e63b4551..ddaca68579 100644 --- a/packages/worker/src/app/handlers/account-email.ts +++ b/packages/worker/src/app/handlers/account-email.ts @@ -45,6 +45,7 @@ export function createAccountEmailHandler(env: Env) { request, user, pathMessageId: readPathMessageId(params), + skipCutoverExposureRecording: true, }) return renderAppPage({ request, diff --git a/packages/worker/src/email/mailbox-read-cutover.node.test.ts b/packages/worker/src/email/mailbox-read-cutover.node.test.ts index 9df07d6c06..20f43880b9 100644 --- a/packages/worker/src/email/mailbox-read-cutover.node.test.ts +++ b/packages/worker/src/email/mailbox-read-cutover.node.test.ts @@ -3,6 +3,7 @@ import { getOwnerEmailAttachmentById, getOwnerEmailMessageById, isMailboxReadCutoverEnabled, + listOwnerEmailAttachmentsForMessage, listOwnerEmailDeliveryEvents, listOwnerEmailMessages, listOwnerEmailMessagesPage, @@ -25,7 +26,7 @@ const { listEmailMessagesMock, searchEmailMessagesMock, listEmailMessagesPageForUserMock, - listEmailAttachmentsForMessageMock, + listEmailAttachmentsForUserMessageMock, getEmailAttachmentRecordByIdMock, listEmailDeliveryEventsMock, loadEmailAttachmentContentMock, @@ -35,7 +36,7 @@ const { listEmailMessagesMock: vi.fn(), searchEmailMessagesMock: vi.fn(), listEmailMessagesPageForUserMock: vi.fn(), - listEmailAttachmentsForMessageMock: vi.fn(), + listEmailAttachmentsForUserMessageMock: vi.fn(), getEmailAttachmentRecordByIdMock: vi.fn(), listEmailDeliveryEventsMock: vi.fn(), loadEmailAttachmentContentMock: vi.fn(), @@ -51,8 +52,8 @@ vi.mock('./repo.ts', () => ({ searchEmailMessagesMock(...args), listEmailMessagesPageForUser: (...args: Array) => listEmailMessagesPageForUserMock(...args), - listEmailAttachmentsForMessage: (...args: Array) => - listEmailAttachmentsForMessageMock(...args), + listEmailAttachmentsForUserMessage: (...args: Array) => + listEmailAttachmentsForUserMessageMock(...args), getEmailAttachmentRecordById: (...args: Array) => getEmailAttachmentRecordByIdMock(...args), listEmailDeliveryEvents: (...args: Array) => @@ -636,6 +637,60 @@ test('mailbox-read-cutover does not fall back from DO and isolates cross-owner i skipExposureRecording: true, }), ).resolves.toBe(false) + + const attachment = { + id: 'att-1', + messageId: 'msg-1', + filename: 'note.txt', + contentType: 'text/plain', + contentId: null, + disposition: 'attachment', + size: 12, + storageKind: 'inline' as const, + storageKey: null, + createdAt: '2026-07-01T12:00:00.000Z', + } + listEmailAttachmentsForUserMessageMock.mockClear() + listEmailAttachmentsForUserMessageMock.mockResolvedValue([]) + await expect( + listOwnerEmailAttachmentsForMessage({ + env: { + ...fakeMailboxEnv({ listAttachmentsForMessage: vi.fn() }).env, + APP_DB: parityErrorDb, + }, + dbUserId, + stableUserId: 'user-other', + messageId: 'msg-1', + now: nowIso, + skipExposureRecording: true, + }), + ).resolves.toEqual([]) + expect(listEmailAttachmentsForUserMessageMock).toHaveBeenCalledWith({ + db: parityErrorDb, + userId: 'user-other', + messageId: 'msg-1', + }) + + listEmailAttachmentsForUserMessageMock.mockClear() + listEmailAttachmentsForUserMessageMock.mockResolvedValue([attachment]) + await expect( + listOwnerEmailAttachmentsForMessage({ + env: { + ...fakeMailboxEnv({ listAttachmentsForMessage: vi.fn() }).env, + APP_DB: parityErrorDb, + }, + dbUserId, + stableUserId, + messageId: 'msg-1', + now: nowIso, + skipExposureRecording: true, + }), + ).resolves.toEqual([attachment]) + expect(listEmailAttachmentsForUserMessageMock).toHaveBeenCalledWith({ + db: parityErrorDb, + userId: stableUserId, + messageId: 'msg-1', + }) }) test('mailbox-read-cutover fails closed on invalid now and future parity timestamps', async () => { diff --git a/packages/worker/src/email/mailbox-read-cutover.ts b/packages/worker/src/email/mailbox-read-cutover.ts index 133240b1c0..eb52ab1265 100644 --- a/packages/worker/src/email/mailbox-read-cutover.ts +++ b/packages/worker/src/email/mailbox-read-cutover.ts @@ -36,7 +36,7 @@ import { import { getEmailAttachmentRecordById, getEmailMessageById, - listEmailAttachmentsForMessage, + listEmailAttachmentsForUserMessage, listEmailDeliveryEvents, listEmailMessages, listEmailMessagesPageForUser, @@ -472,8 +472,9 @@ export async function listOwnerEmailAttachmentsForMessage( ): Promise> { const useMailbox = await useMailboxReads(input) if (!useMailbox) { - return listEmailAttachmentsForMessage({ + return listEmailAttachmentsForUserMessage({ db: input.env.APP_DB, + userId: input.stableUserId, messageId: input.messageId, }) } diff --git a/packages/worker/src/email/repo-attachments.workers.test.ts b/packages/worker/src/email/repo-attachments.workers.test.ts new file mode 100644 index 0000000000..3b4cdb9298 --- /dev/null +++ b/packages/worker/src/email/repo-attachments.workers.test.ts @@ -0,0 +1,60 @@ +import { env } from 'cloudflare:workers' +import { expect, test } from 'vitest' +import { listEmailAttachmentsForUserMessage } from './repo.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +async function insertMessageWithAttachment(input: { + userId: string + messageId: string + attachmentId: string +}) { + const createdAt = '2026-07-01T12:00:00.000Z' + await env.APP_DB.prepare( + `INSERT INTO email_messages ( + id, direction, user_id, from_address, subject, processing_status, + created_at, updated_at + ) VALUES (?, 'inbound', ?, 'sender@example.com', 'Hello', 'stored', ?, ?)`, + ) + .bind(input.messageId, input.userId, createdAt, createdAt) + .run() + await env.APP_DB.prepare( + `INSERT INTO email_attachments ( + id, message_id, filename, content_type, size, storage_kind, created_at + ) VALUES (?, ?, 'note.txt', 'text/plain', 12, 'unavailable', ?)`, + ) + .bind(input.attachmentId, input.messageId, createdAt) + .run() +} + +test('listEmailAttachmentsForUserMessage scopes attachments to message owner', async () => { + await ensureEmailTestSchema(env.APP_DB) + const ownerId = `owner-${crypto.randomUUID()}` + const otherId = `other-${crypto.randomUUID()}` + const messageId = `msg-${crypto.randomUUID()}` + const attachmentId = `att-${crypto.randomUUID()}` + await insertMessageWithAttachment({ + userId: ownerId, + messageId, + attachmentId, + }) + + const ownerAttachments = await listEmailAttachmentsForUserMessage({ + db: env.APP_DB, + userId: ownerId, + messageId, + }) + expect(ownerAttachments).toEqual([ + expect.objectContaining({ + id: attachmentId, + messageId, + filename: 'note.txt', + }), + ]) + + const crossOwnerAttachments = await listEmailAttachmentsForUserMessage({ + db: env.APP_DB, + userId: otherId, + messageId, + }) + expect(crossOwnerAttachments).toEqual([]) +}) diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 7027dd55f5..10ada76802 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -1362,6 +1362,25 @@ export async function listEmailAttachmentsForMessage(input: { return (result.results ?? []).map(mapAttachmentRow) } +export async function listEmailAttachmentsForUserMessage(input: { + db: D1Database + userId: string + messageId: string +}) { + const result = await input.db + .prepare( + `SELECT attachment.* + FROM email_attachments attachment + JOIN email_messages message ON message.id = attachment.message_id + WHERE attachment.message_id = ? + AND message.user_id = ? + ORDER BY attachment.created_at ASC, attachment.id ASC`, + ) + .bind(input.messageId, input.userId) + .all>() + return (result.results ?? []).map(mapAttachmentRow) +} + export async function getEmailAttachmentRecordById(input: { db: D1Database userId: string From 3b030b3342d39bb241b3d087aa52e06af3266793 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 02:45:30 +0000 Subject: [PATCH 42/62] test(email): use typed account email mock Co-authored-by: Kent C. Dodds --- .../src/app/handlers/account-email-exposure.node.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/worker/src/app/handlers/account-email-exposure.node.test.ts b/packages/worker/src/app/handlers/account-email-exposure.node.test.ts index 0588dad751..c954cfd66a 100644 --- a/packages/worker/src/app/handlers/account-email-exposure.node.test.ts +++ b/packages/worker/src/app/handlers/account-email-exposure.node.test.ts @@ -1,4 +1,5 @@ import { expect, test, vi } from 'vitest' +import type * as AccountEmailData from '#app/account-email-data.ts' const loadAccountEmailDataMock = vi.hoisted(() => vi.fn(async () => ({ @@ -21,8 +22,7 @@ const loadAccountEmailDataMock = vi.hoisted(() => ) vi.mock('#app/account-email-data.ts', async (importOriginal) => { - const actual = - await importOriginal() + const actual = await importOriginal() return { ...actual, loadAccountEmailData: (...args: Array) => From e72ce799e60f293eea80bc4532f47164fd72e84d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 04:39:17 +0000 Subject: [PATCH 43/62] feat(email): add outbound provider reverse index Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 34 +- .../0128-email-outbound-provider-index.sql | 43 ++ packages/worker/src/account/data-targets.ts | 8 + packages/worker/src/account/export.ts | 3 + .../admin/mailbox-maintenance.node.test.ts | 10 + .../worker/src/app/retention.node.test.ts | 40 ++ packages/worker/src/app/retention.ts | 5 + ...ound-provider-index-migration.node.test.ts | 149 +++++++ .../src/email/outbound-provider-index.ts | 402 ++++++++++++++++++ .../outbound-provider-index.workers.test.ts | 380 +++++++++++++++++ packages/worker/src/email/outbound.ts | 2 + packages/worker/src/email/repo.ts | 104 ++++- packages/worker/src/email/service.ts | 5 + packages/worker/src/email/system-email.ts | 9 +- packages/worker/src/email/test-schema.ts | 15 + tools/migration-ledger.json | 4 + 16 files changed, 1183 insertions(+), 30 deletions(-) create mode 100644 packages/worker/migrations/0128-email-outbound-provider-index.sql create mode 100644 packages/worker/src/email/outbound-provider-index-migration.node.test.ts create mode 100644 packages/worker/src/email/outbound-provider-index.ts create mode 100644 packages/worker/src/email/outbound-provider-index.workers.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index e8fb5ce933..ce81ea0763 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -1115,10 +1115,25 @@ below. Mailbox objects. - **Low-write email config** — sender identities, inboxes, inbox addresses, sender rules, and similar low-churn configuration stay in D1. -- **Provider-message reverse lookup** — outbound `provider_message_id` → owner - resolution stays in D1 until the delivery webhook path already knows the - owning user. Contextless provider-id reverse lookups must not require - enumerating per-user Mailbox objects. +- **Provider-message reverse lookup** — outbound Cloudflare sending webhooks + resolve owner/message through the derived D1 table + `email_outbound_provider_index` (migration + `0128-email-outbound-provider-index.sql`), keyed by + `(provider, provider_message_id)` with `user_id`, `message_id`, `inbox_id`, + and created/updated timestamps (indexes on `user_id` and unique `message_id`). + `email_messages.provider_message_id` remains authoritative: outbound terminal + success / provider-id updates dual-write the index after the message row + write; `deleteEmailMessageById`, user/system retention, and account deletion + remove owner index rows. Account export treats the table as derived global + lookup (`includeInExport: false` / + `derivedData.email_outbound_provider_index`) because authoritative outbound + message rows are already exported. `recordProviderEmailDeliveryEvent` resolves + index-first, then loads the owner-scoped message by `user_id`/`message_id` (no + full-table provider scan). `system:email` rows are indexed when they carry a + provider id. Read helper `loadOutboundProviderIndexParityReport` compares + linked outbound messages vs index counts/missing/mismatched for production + evidence. This phase does not flip Mailbox write authority; contextless + provider-id reverse lookups must not enumerate per-user Mailbox objects. ### Inbound durability boundary (D1-authoritative dual-write) @@ -1738,11 +1753,12 @@ Current retention policies: batches. Retention deletes the deterministic `emailRawMimeKey(userId, messageId)` from `EMAIL_BLOBS` before D1 rows; if the blob delete fails, those rows are skipped and still selected on later runs so - cleanup can retry. Dependent `email_attachments` rows are deleted before their - messages, and threads left with no messages are pruned for the affected users. - After Mailbox cut-over, the same 365-day window and blob-before-row ordering - are self-enforced by the Mailbox DO alarm; `system:email` stays on the D1 - system-email retention job. + cleanup can retry. Dependent `email_attachments` rows and derived + `email_outbound_provider_index` rows are deleted before their messages, and + threads left with no messages are pruned for the affected users. After Mailbox + cut-over, the same 365-day window and blob-before-row ordering are + self-enforced by the Mailbox DO alarm; `system:email` stays on the D1 + system-email retention job (and clears matching provider-index rows). - `entitlement_daily_counters`: **retired** — dropped by migration `0126-drop-entitlement-daily-counters.sql` after stages 1/2 stopped mirror writes and detached runtime inventory. No scheduled retention disposition or diff --git a/packages/worker/migrations/0128-email-outbound-provider-index.sql b/packages/worker/migrations/0128-email-outbound-provider-index.sql new file mode 100644 index 0000000000..2683060ba1 --- /dev/null +++ b/packages/worker/migrations/0128-email-outbound-provider-index.sql @@ -0,0 +1,43 @@ +-- Global outbound provider → owner reverse index. +-- email_messages remains authoritative for provider_message_id; this table is a +-- derived lookup so contextless delivery webhooks can resolve the owning user +-- without a full-table provider_message_id scan (and without enumerating +-- per-user Mailbox objects). Backfill includes system:email when present. + +CREATE TABLE email_outbound_provider_index ( + provider TEXT NOT NULL, + provider_message_id TEXT NOT NULL, + user_id TEXT NOT NULL, + message_id TEXT NOT NULL, + inbox_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (provider, provider_message_id) +); + +CREATE INDEX idx_email_outbound_provider_index_user_id + ON email_outbound_provider_index(user_id); + +CREATE UNIQUE INDEX idx_email_outbound_provider_index_message_id + ON email_outbound_provider_index(message_id); + +INSERT INTO email_outbound_provider_index ( + provider, + provider_message_id, + user_id, + message_id, + inbox_id, + created_at, + updated_at +) +SELECT + 'cloudflare-email', + provider_message_id, + user_id, + id, + inbox_id, + created_at, + updated_at +FROM email_messages +WHERE direction = 'outbound' + AND provider_message_id IS NOT NULL; diff --git a/packages/worker/src/account/data-targets.ts b/packages/worker/src/account/data-targets.ts index 3b918eabc8..03c412d5ed 100644 --- a/packages/worker/src/account/data-targets.ts +++ b/packages/worker/src/account/data-targets.ts @@ -240,6 +240,14 @@ export const accountUserDataTargets: ReadonlyArray = [ { kind: 'user_id', table: 'entity_sources' }, { kind: 'user_id', table: 'email_delivery_events' }, { kind: 'attachment_parent', table: 'email_attachments' }, + { + kind: 'user_id', + table: 'email_outbound_provider_index', + includeInExport: false, + surface: 'email_outbound_provider_index', + reason: + 'Derived global provider→owner reverse lookup rebuilt from outbound email_messages.provider_message_id. Account deletion clears owner rows; export already includes the authoritative message rows and intentionally omits this lookup table.', + }, { kind: 'user_id', table: 'email_messages' }, { kind: 'user_id', table: 'email_threads' }, { kind: 'user_id', table: 'email_inbox_addresses' }, diff --git a/packages/worker/src/account/export.ts b/packages/worker/src/account/export.ts index 7e6f9176b7..aec6d6169a 100644 --- a/packages/worker/src/account/export.ts +++ b/packages/worker/src/account/export.ts @@ -178,6 +178,7 @@ export type AccountExportManifest = { derivedData: { vectorize: string r2: string + email_outbound_provider_index: string } excludedDurableObjects: Array<{ name: string @@ -1756,6 +1757,8 @@ function buildManifest(input: { vectorize: 'Vectorize entries for memories, jobs, and packages are derived from exported D1 rows and are intentionally excluded; rebuild them by reindexing after import.', r2: 'R2 raw MIME, attachment, avatar, and icon bytes are exported through the r2_object section in bounded 256 KiB base64 chunks. Missing objects are returned explicitly instead of being silently omitted.', + email_outbound_provider_index: + 'Derived global provider→owner reverse lookup for outbound delivery webhooks. Rebuilt from exported outbound email_messages.provider_message_id rows; omitted from the D1 export payload.', }, excludedDurableObjects: getAccountExportExcludedDurableObjects(), excludedD1Surfaces: getAccountExportExcludedD1Surfaces(), diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts index a4085d79ef..56779f3ad3 100644 --- a/packages/worker/src/admin/mailbox-maintenance.node.test.ts +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -235,6 +235,16 @@ function createDeleteMessageDb() { created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); + CREATE TABLE email_outbound_provider_index ( + provider TEXT NOT NULL, + provider_message_id TEXT NOT NULL, + user_id TEXT NOT NULL, + message_id TEXT NOT NULL, + inbox_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (provider, provider_message_id) + ); CREATE TABLE email_attachments ( id TEXT PRIMARY KEY, message_id TEXT NOT NULL, diff --git a/packages/worker/src/app/retention.node.test.ts b/packages/worker/src/app/retention.node.test.ts index b5a18cd718..e0fecdbca5 100644 --- a/packages/worker/src/app/retention.node.test.ts +++ b/packages/worker/src/app/retention.node.test.ts @@ -189,6 +189,16 @@ function createRetentionDb() { created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); + CREATE TABLE email_outbound_provider_index ( + provider TEXT NOT NULL, + provider_message_id TEXT NOT NULL, + user_id TEXT NOT NULL, + message_id TEXT NOT NULL, + inbox_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (provider, provider_message_id) + ); CREATE TABLE email_attachments ( id TEXT PRIMARY KEY, message_id TEXT NOT NULL, @@ -595,6 +605,25 @@ test('email message retention deletes old user rows, R2 blobs, attachments, and 'email-attachment:v1:user-1/msg-old-plain/att-ext', ?)`, ) .run(daysAgo(emailMessageRetentionDays + 1)) + const indexCreatedAt = daysAgo(emailMessageRetentionDays + 1) + sqlite + .prepare( + `INSERT INTO email_outbound_provider_index ( + provider, provider_message_id, user_id, message_id, inbox_id, + created_at, updated_at + ) VALUES + ('cloudflare-email', 'prov-old-blob', 'user-1', 'msg-old-blob', NULL, ?, ?), + ('cloudflare-email', 'prov-fresh', 'user-1', 'msg-fresh', NULL, ?, ?), + ('cloudflare-email', 'prov-system', 'system:email', 'msg-old-system', NULL, ?, ?)`, + ) + .run( + indexCreatedAt, + indexCreatedAt, + daysAgo(1), + daysAgo(1), + indexCreatedAt, + indexCreatedAt, + ) const blobDelete = vi.fn(async () => undefined) const result = await pruneUserEmailMessagesForRetention({ @@ -625,6 +654,17 @@ test('email message retention deletes old user rows, R2 blobs, attachments, and ]) expect(idsForTable(sqlite, 'email_attachments')).toEqual([]) expect(idsForTable(sqlite, 'email_threads')).toEqual(['thread-mixed']) + expect( + ( + sqlite + .prepare( + `SELECT provider_message_id + FROM email_outbound_provider_index + ORDER BY provider_message_id`, + ) + .all() as Array<{ provider_message_id: string }> + ).map((row) => row.provider_message_id), + ).toEqual(['prov-fresh', 'prov-system']) }) test('email message retention never deletes rows whose blob cannot be deleted first', async () => { diff --git a/packages/worker/src/app/retention.ts b/packages/worker/src/app/retention.ts index cdcaaa2b6d..a97c0c0e37 100644 --- a/packages/worker/src/app/retention.ts +++ b/packages/worker/src/app/retention.ts @@ -1,4 +1,5 @@ import { accountRetentionDispositions } from '#app/account-retention-dispositions.ts' +import { deleteOutboundProviderIndexByMessageIds } from '#worker/email/outbound-provider-index.ts' import { emailRawMimeKey } from '#worker/email/repo.ts' import { systemEmailOwnerId } from '#worker/email/system-email.ts' import { runD1WithRetry } from '#worker/d1-retry.ts' @@ -711,6 +712,10 @@ export async function pruneUserEmailMessagesForRetention(input: { idColumn: 'message_id', ids: messageIds, }) + await deleteOutboundProviderIndexByMessageIds({ + db: input.db, + messageIds: messageIds.map(String), + }) result.deletedMessages = await deleteByIds({ db: input.db, table: 'email_messages', diff --git a/packages/worker/src/email/outbound-provider-index-migration.node.test.ts b/packages/worker/src/email/outbound-provider-index-migration.node.test.ts new file mode 100644 index 0000000000..8bdf99e29c --- /dev/null +++ b/packages/worker/src/email/outbound-provider-index-migration.node.test.ts @@ -0,0 +1,149 @@ +import { readFileSync } from 'node:fs' +import { DatabaseSync } from 'node:sqlite' +import { expect, test } from 'vitest' +import { emailOutboundProviderCloudflare } from './outbound-provider-index.ts' + +const migrationUrl = new URL( + '../../migrations/0128-email-outbound-provider-index.sql', + import.meta.url, +) + +function createPreMigrationDb() { + const db = new DatabaseSync(':memory:') + db.exec(` + CREATE TABLE email_messages ( + id TEXT PRIMARY KEY, + direction TEXT NOT NULL, + user_id TEXT NOT NULL, + inbox_id TEXT, + provider_message_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE UNIQUE INDEX idx_email_messages_provider_message_id + ON email_messages(provider_message_id) + WHERE direction = 'outbound' AND provider_message_id IS NOT NULL; + `) + return db +} + +test('provider index migration backfills outbound provider ids including system owner', () => { + const db = createPreMigrationDb() + db.exec(` + INSERT INTO email_messages ( + id, direction, user_id, inbox_id, provider_message_id, created_at, updated_at + ) VALUES + ('user-msg', 'outbound', 'user-1', 'inbox-1', 'prov-user', + '2026-08-01T00:00:00.000Z', '2026-08-01T00:00:00.000Z'), + ('system-msg', 'outbound', 'system:email', NULL, 'prov-system', + '2026-08-01T00:01:00.000Z', '2026-08-01T00:01:00.000Z'), + ('inbound-msg', 'inbound', 'user-1', 'inbox-1', 'prov-inbound', + '2026-08-01T00:02:00.000Z', '2026-08-01T00:02:00.000Z'), + ('no-provider', 'outbound', 'user-1', NULL, NULL, + '2026-08-01T00:03:00.000Z', '2026-08-01T00:03:00.000Z'); + `) + + db.exec(readFileSync(migrationUrl, 'utf8')) + + expect( + db + .prepare( + `SELECT provider, provider_message_id, user_id, message_id, inbox_id + FROM email_outbound_provider_index + ORDER BY provider_message_id ASC`, + ) + .all(), + ).toEqual([ + { + provider: emailOutboundProviderCloudflare, + provider_message_id: 'prov-system', + user_id: 'system:email', + message_id: 'system-msg', + inbox_id: null, + }, + { + provider: emailOutboundProviderCloudflare, + provider_message_id: 'prov-user', + user_id: 'user-1', + message_id: 'user-msg', + inbox_id: 'inbox-1', + }, + ]) + + expect(() => { + db.prepare( + `INSERT INTO email_outbound_provider_index ( + provider, provider_message_id, user_id, message_id, inbox_id, + created_at, updated_at + ) VALUES (?, ?, ?, ?, NULL, ?, ?)`, + ).run( + emailOutboundProviderCloudflare, + 'prov-user', + 'other-user', + 'other-msg', + '2026-08-01T00:04:00.000Z', + '2026-08-01T00:04:00.000Z', + ) + }).toThrow(/UNIQUE constraint failed/) + + expect(() => { + db.prepare( + `INSERT INTO email_outbound_provider_index ( + provider, provider_message_id, user_id, message_id, inbox_id, + created_at, updated_at + ) VALUES (?, ?, ?, ?, NULL, ?, ?)`, + ).run( + emailOutboundProviderCloudflare, + 'prov-other', + 'user-1', + 'user-msg', + '2026-08-01T00:04:00.000Z', + '2026-08-01T00:04:00.000Z', + ) + }).toThrow(/UNIQUE constraint failed/) +}) + +test('provider index migration backfill counts match linked outbound messages', () => { + const db = createPreMigrationDb() + db.exec(` + INSERT INTO email_messages ( + id, direction, user_id, inbox_id, provider_message_id, created_at, updated_at + ) VALUES + ('a', 'outbound', 'user-1', NULL, 'p-a', + '2026-08-01T00:00:00.000Z', '2026-08-01T00:00:00.000Z'), + ('b', 'outbound', 'system:email', NULL, 'p-b', + '2026-08-01T00:01:00.000Z', '2026-08-01T00:01:00.000Z'); + `) + db.exec(readFileSync(migrationUrl, 'utf8')) + + const linked = db + .prepare( + `SELECT COUNT(*) AS count + FROM email_messages + WHERE direction = 'outbound' AND provider_message_id IS NOT NULL`, + ) + .get() as { count: number } + const indexed = db + .prepare(`SELECT COUNT(*) AS count FROM email_outbound_provider_index`) + .get() as { count: number } + const missingFromIndex = db + .prepare( + `SELECT COUNT(*) AS count + FROM email_messages AS message + WHERE message.direction = 'outbound' + AND message.provider_message_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM email_outbound_provider_index AS idx + WHERE idx.provider = ? + AND idx.provider_message_id = message.provider_message_id + AND idx.message_id = message.id + AND idx.user_id = message.user_id + )`, + ) + .get(emailOutboundProviderCloudflare) as { count: number } + + expect(linked.count).toBe(2) + expect(indexed.count).toBe(2) + expect(missingFromIndex.count).toBe(0) +}) diff --git a/packages/worker/src/email/outbound-provider-index.ts b/packages/worker/src/email/outbound-provider-index.ts new file mode 100644 index 0000000000..b3f9f8d7b3 --- /dev/null +++ b/packages/worker/src/email/outbound-provider-index.ts @@ -0,0 +1,402 @@ +/** + * Derived D1 reverse index: outbound provider message id → owning message. + * `email_messages.provider_message_id` stays authoritative; this table exists + * so contextless provider webhooks resolve owner/message without scanning the + * full messages table or enumerating Mailbox objects. + */ + +export const emailOutboundProviderCloudflare = 'cloudflare-email' + +export type EmailOutboundProviderIndexRow = { + provider: string + providerMessageId: string + userId: string + messageId: string + inboxId: string | null + createdAt: string + updatedAt: string +} + +function mapIndexRow( + row: Record, +): EmailOutboundProviderIndexRow { + return { + provider: String(row['provider']), + providerMessageId: String(row['provider_message_id']), + userId: String(row['user_id']), + messageId: String(row['message_id']), + inboxId: row['inbox_id'] == null ? null : String(row['inbox_id']), + createdAt: String(row['created_at']), + updatedAt: String(row['updated_at']), + } +} + +export async function getOutboundProviderIndexRow(input: { + db: D1Database + provider?: string + providerMessageId: string +}): Promise { + const provider = input.provider ?? emailOutboundProviderCloudflare + const row = await input.db + .prepare( + `SELECT * + FROM email_outbound_provider_index + WHERE provider = ? + AND provider_message_id = ? + LIMIT 1`, + ) + .bind(provider, input.providerMessageId) + .first>() + return row ? mapIndexRow(row) : null +} + +/** + * Upsert the derived index for one outbound message. Callers must write the + * authoritative `email_messages` row first. Replaces any prior mapping for the + * same `message_id` when the provider id changes. + */ +export async function upsertOutboundProviderIndex(input: { + db: D1Database + provider?: string + providerMessageId: string + userId: string + messageId: string + inboxId?: string | null + now?: string +}): Promise { + const provider = input.provider ?? emailOutboundProviderCloudflare + const now = input.now ?? new Date().toISOString() + await input.db.batch([ + input.db + .prepare( + `DELETE FROM email_outbound_provider_index + WHERE message_id = ? + AND NOT (provider = ? AND provider_message_id = ?)`, + ) + .bind(input.messageId, provider, input.providerMessageId), + input.db + .prepare( + `INSERT INTO email_outbound_provider_index ( + provider, provider_message_id, user_id, message_id, inbox_id, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(provider, provider_message_id) DO UPDATE SET + user_id = excluded.user_id, + message_id = excluded.message_id, + inbox_id = excluded.inbox_id, + updated_at = excluded.updated_at`, + ) + .bind( + provider, + input.providerMessageId, + input.userId, + input.messageId, + input.inboxId ?? null, + now, + now, + ), + ]) +} + +export async function deleteOutboundProviderIndexByMessageId(input: { + db: D1Database + messageId: string +}): Promise { + const result = await input.db + .prepare( + `DELETE FROM email_outbound_provider_index + WHERE message_id = ?`, + ) + .bind(input.messageId) + .run() + return Number(result.meta.changes ?? 0) +} + +export async function deleteOutboundProviderIndexByMessageIds(input: { + db: D1Database + messageIds: ReadonlyArray +}): Promise { + if (input.messageIds.length === 0) return 0 + let deleted = 0 + const chunkSize = 100 + for (let index = 0; index < input.messageIds.length; index += chunkSize) { + const chunk = input.messageIds.slice(index, index + chunkSize) + const placeholders = chunk.map(() => '?').join(', ') + const result = await input.db + .prepare( + `DELETE FROM email_outbound_provider_index + WHERE message_id IN (${placeholders})`, + ) + .bind(...chunk) + .run() + deleted += Number(result.meta.changes ?? 0) + } + return deleted +} + +export type OutboundProviderIndexParityMismatch = { + kind: 'missing_from_index' | 'missing_from_messages' | 'mismatched' + provider: string + providerMessageId: string + userId: string + messageId: string + detail: string +} + +export type OutboundProviderIndexParityReport = { + linkedMessageCount: number + indexCount: number + missingFromIndexCount: number + missingFromMessagesCount: number + mismatchedCount: number + sampleMismatches: Array + parity: boolean +} + +/** + * Read-only compare of outbound messages that carry a provider id versus the + * derived reverse index. Optional `userId` scopes the report to one owner + * (including `system:email`). + */ +export async function loadOutboundProviderIndexParityReport(input: { + db: D1Database + userId?: string + sampleLimit?: number +}): Promise { + const sampleLimit = input.sampleLimit ?? 20 + const userFilter = input.userId == null ? '' : ' AND user_id = ?' + const userBindings = input.userId == null ? [] : [input.userId] + + const linkedMessageCount = Number( + ( + await input.db + .prepare( + `SELECT COUNT(*) AS count + FROM email_messages + WHERE direction = 'outbound' + AND provider_message_id IS NOT NULL${userFilter}`, + ) + .bind(...userBindings) + .first<{ count: number }>() + )?.count ?? 0, + ) + const indexCount = Number( + ( + await input.db + .prepare( + `SELECT COUNT(*) AS count + FROM email_outbound_provider_index + WHERE 1 = 1${userFilter}`, + ) + .bind(...userBindings) + .first<{ count: number }>() + )?.count ?? 0, + ) + + const missingFromIndexRows = await input.db + .prepare( + `SELECT + message.id AS message_id, + message.user_id AS user_id, + message.provider_message_id AS provider_message_id + FROM email_messages AS message + WHERE message.direction = 'outbound' + AND message.provider_message_id IS NOT NULL${ + input.userId == null ? '' : ' AND message.user_id = ?' + } + AND NOT EXISTS ( + SELECT 1 + FROM email_outbound_provider_index AS idx + WHERE idx.provider = ? + AND idx.provider_message_id = message.provider_message_id + AND idx.message_id = message.id + AND idx.user_id = message.user_id + ) + ORDER BY message.created_at ASC, message.id ASC + LIMIT ?`, + ) + .bind( + ...(input.userId == null ? [] : [input.userId]), + emailOutboundProviderCloudflare, + sampleLimit, + ) + .all<{ + message_id: string + user_id: string + provider_message_id: string + }>() + + const missingFromMessagesRows = await input.db + .prepare( + `SELECT + idx.provider AS provider, + idx.provider_message_id AS provider_message_id, + idx.user_id AS user_id, + idx.message_id AS message_id + FROM email_outbound_provider_index AS idx + WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} + AND NOT EXISTS ( + SELECT 1 + FROM email_messages AS message + WHERE message.id = idx.message_id + AND message.user_id = idx.user_id + AND message.direction = 'outbound' + AND message.provider_message_id = idx.provider_message_id + ) + ORDER BY idx.created_at ASC, idx.message_id ASC + LIMIT ?`, + ) + .bind(...(input.userId == null ? [] : [input.userId]), sampleLimit) + .all<{ + provider: string + provider_message_id: string + user_id: string + message_id: string + }>() + + const mismatchedRows = await input.db + .prepare( + `SELECT + idx.provider AS provider, + idx.provider_message_id AS provider_message_id, + idx.user_id AS index_user_id, + idx.message_id AS message_id, + message.user_id AS message_user_id, + message.provider_message_id AS message_provider_message_id, + message.inbox_id AS message_inbox_id, + idx.inbox_id AS index_inbox_id + FROM email_outbound_provider_index AS idx + JOIN email_messages AS message + ON message.id = idx.message_id + WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} + AND ( + message.user_id != idx.user_id + OR message.direction != 'outbound' + OR message.provider_message_id IS NULL + OR message.provider_message_id != idx.provider_message_id + OR IFNULL(message.inbox_id, '') != IFNULL(idx.inbox_id, '') + ) + ORDER BY idx.created_at ASC, idx.message_id ASC + LIMIT ?`, + ) + .bind(...(input.userId == null ? [] : [input.userId]), sampleLimit) + .all<{ + provider: string + provider_message_id: string + index_user_id: string + message_id: string + message_user_id: string + message_provider_message_id: string | null + message_inbox_id: string | null + index_inbox_id: string | null + }>() + + const missingFromIndexCount = Number( + ( + await input.db + .prepare( + `SELECT COUNT(*) AS count + FROM email_messages AS message + WHERE message.direction = 'outbound' + AND message.provider_message_id IS NOT NULL${ + input.userId == null ? '' : ' AND message.user_id = ?' + } + AND NOT EXISTS ( + SELECT 1 + FROM email_outbound_provider_index AS idx + WHERE idx.provider = ? + AND idx.provider_message_id = message.provider_message_id + AND idx.message_id = message.id + AND idx.user_id = message.user_id + )`, + ) + .bind( + ...(input.userId == null ? [] : [input.userId]), + emailOutboundProviderCloudflare, + ) + .first<{ count: number }>() + )?.count ?? 0, + ) + const missingFromMessagesCount = Number( + ( + await input.db + .prepare( + `SELECT COUNT(*) AS count + FROM email_outbound_provider_index AS idx + WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} + AND NOT EXISTS ( + SELECT 1 + FROM email_messages AS message + WHERE message.id = idx.message_id + AND message.user_id = idx.user_id + AND message.direction = 'outbound' + AND message.provider_message_id = idx.provider_message_id + )`, + ) + .bind(...(input.userId == null ? [] : [input.userId])) + .first<{ count: number }>() + )?.count ?? 0, + ) + const mismatchedCount = Number( + ( + await input.db + .prepare( + `SELECT COUNT(*) AS count + FROM email_outbound_provider_index AS idx + JOIN email_messages AS message + ON message.id = idx.message_id + WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} + AND ( + message.user_id != idx.user_id + OR message.direction != 'outbound' + OR message.provider_message_id IS NULL + OR message.provider_message_id != idx.provider_message_id + OR IFNULL(message.inbox_id, '') != IFNULL(idx.inbox_id, '') + )`, + ) + .bind(...(input.userId == null ? [] : [input.userId])) + .first<{ count: number }>() + )?.count ?? 0, + ) + + const sampleMismatches: Array = [ + ...(missingFromIndexRows.results ?? []).map((row) => ({ + kind: 'missing_from_index' as const, + provider: emailOutboundProviderCloudflare, + providerMessageId: row.provider_message_id, + userId: row.user_id, + messageId: row.message_id, + detail: 'Linked outbound message has no matching index row.', + })), + ...(missingFromMessagesRows.results ?? []).map((row) => ({ + kind: 'missing_from_messages' as const, + provider: row.provider, + providerMessageId: row.provider_message_id, + userId: row.user_id, + messageId: row.message_id, + detail: 'Index row has no matching linked outbound message.', + })), + ...(mismatchedRows.results ?? []).map((row) => ({ + kind: 'mismatched' as const, + provider: row.provider, + providerMessageId: row.provider_message_id, + userId: row.index_user_id, + messageId: row.message_id, + detail: `Index/message fields diverge (message user=${row.message_user_id}, provider_message_id=${row.message_provider_message_id ?? 'null'}, inbox=${row.message_inbox_id ?? 'null'} vs index inbox=${row.index_inbox_id ?? 'null'}).`, + })), + ].slice(0, sampleLimit) + + return { + linkedMessageCount, + indexCount, + missingFromIndexCount, + missingFromMessagesCount, + mismatchedCount, + sampleMismatches, + parity: + linkedMessageCount === indexCount && + missingFromIndexCount === 0 && + missingFromMessagesCount === 0 && + mismatchedCount === 0, + } +} diff --git a/packages/worker/src/email/outbound-provider-index.workers.test.ts b/packages/worker/src/email/outbound-provider-index.workers.test.ts new file mode 100644 index 0000000000..2479a775b0 --- /dev/null +++ b/packages/worker/src/email/outbound-provider-index.workers.test.ts @@ -0,0 +1,380 @@ +import { env } from 'cloudflare:workers' +import { expect, test } from 'vitest' +import { createStableUserIdFromEmail } from '#worker/user-id.ts' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { processCloudflareEmailDeliveryEvent } from './delivery-events.ts' +import { + deleteOutboundProviderIndexByMessageId, + emailOutboundProviderCloudflare, + getOutboundProviderIndexRow, + loadOutboundProviderIndexParityReport, +} from './outbound-provider-index.ts' +import { sendOutboundEmail } from './outbound.ts' +import { + deleteEmailMessageById, + getOutboundEmailMessageByProviderMessageId, + insertEmailMessage, + updateEmailMessageDelivery, +} from './repo.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +const platformBaseUrl = 'https://kody.example.com' + +function createBindingSendEnv() { + return { + ...env, + APP_BASE_URL: platformBaseUrl, + EMAIL: { + async send() { + return { messageId: `provider-send-${crypto.randomUUID()}` } + }, + }, + } +} + +async function seedVerifiedAccount(email: string) { + const username = `idx-send-${crypto.randomUUID().slice(0, 8)}` + const stableUserId = await createStableUserIdFromEmail(email) + await env.APP_DB.prepare( + `INSERT INTO users ( + username, email, password_hash, email_verified_at, plan, stable_user_id + ) VALUES (?, ?, ?, ?, ?, ?)`, + ) + .bind( + username, + email, + 'test-password-hash', + new Date().toISOString(), + 'max', + stableUserId, + ) + .run() + return stableUserId +} + +test('insert and delivery update dual-write the outbound provider index', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = `idx-user-${crypto.randomUUID()}` + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + inboxId: null, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'Index dual-write', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-08-01T12:00:00.000Z', + }, + }) + + expect( + await getOutboundProviderIndexRow({ + db: env.APP_DB, + providerMessageId, + }), + ).toMatchObject({ + provider: emailOutboundProviderCloudflare, + providerMessageId, + userId, + messageId: message.id, + inboxId: null, + }) + + const rotatedProviderMessageId = `provider-rotated-${crypto.randomUUID()}` + await updateEmailMessageDelivery({ + db: env.APP_DB, + messageId: message.id, + status: 'sent', + providerMessageId: rotatedProviderMessageId, + userId, + inboxId: null, + sentAt: '2026-08-01T12:01:00.000Z', + }) + + expect( + await getOutboundProviderIndexRow({ + db: env.APP_DB, + providerMessageId, + }), + ).toBeNull() + expect( + await getOutboundProviderIndexRow({ + db: env.APP_DB, + providerMessageId: rotatedProviderMessageId, + }), + ).toMatchObject({ + messageId: message.id, + userId, + }) + expect( + await loadOutboundProviderIndexParityReport({ + db: env.APP_DB, + userId, + }), + ).toMatchObject({ + linkedMessageCount: 1, + indexCount: 1, + parity: true, + }) +}) + +test('provider delivery lifecycle resolves through the index for user and system owners', async () => { + await ensureEmailTestSchema(env.APP_DB) + + for (const userId of [ + `idx-lifecycle-${crypto.randomUUID()}`, + 'system:email', + ]) { + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'sender@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'Lifecycle', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-08-01T13:00:00.000Z', + }, + }) + + expect( + await getOutboundEmailMessageByProviderMessageId({ + db: env.APP_DB, + providerMessageId, + }), + ).toMatchObject({ id: message.id, userId }) + + const recorded = await processCloudflareEmailDeliveryEvent({ + db: env.APP_DB, + body: { + type: 'cf.email.sending.message.delivered', + source: { + type: 'email.sending', + zoneId: 'zone-1', + domain: 'inbox.example.com', + }, + payload: { + eventId: `event-${crypto.randomUUID()}`, + messageId: providerMessageId, + sender: 'sender@inbox.example.com', + recipient: 'recipient@example.net', + subject: 'Lifecycle', + terminal: true, + delivery: { + status: 'delivered', + provider: 'external_smtp', + smtpStatusCode: '250', + smtpResponse: '250 2.0.0 accepted', + }, + }, + metadata: { + accountId: 'account-1', + eventSubscriptionId: 'subscription-1', + eventSchemaVersion: 1, + eventTimestamp: '2026-08-01T13:02:00.000Z', + }, + }, + }) + expect(recorded.outcome).toBe('recorded') + expect(recorded.message).toMatchObject({ + id: message.id, + userId, + deliveryStatus: 'delivered', + }) + + const duplicate = await processCloudflareEmailDeliveryEvent({ + db: env.APP_DB, + body: { + type: 'cf.email.sending.message.delivered', + source: { + type: 'email.sending', + zoneId: 'zone-1', + domain: 'inbox.example.com', + }, + payload: { + eventId: recorded.event!.providerEventId, + messageId: providerMessageId, + sender: 'sender@inbox.example.com', + recipient: 'recipient@example.net', + terminal: true, + delivery: { + status: 'delivered', + provider: 'external_smtp', + }, + }, + metadata: { + accountId: 'account-1', + eventSubscriptionId: 'subscription-1', + eventSchemaVersion: 1, + eventTimestamp: '2026-08-01T13:02:00.000Z', + }, + }, + }) + expect(duplicate.outcome).toBe('duplicate') + } + + await deleteOutboundProviderIndexByMessageId({ + db: env.APP_DB, + messageId: 'missing', + }) + expect( + ( + await processCloudflareEmailDeliveryEvent({ + db: env.APP_DB, + body: { + type: 'cf.email.sending.message.bounced', + source: { + type: 'email.sending', + zoneId: 'zone-1', + domain: 'inbox.example.com', + }, + payload: { + eventId: `event-${crypto.randomUUID()}`, + messageId: 'unknown-provider-message', + sender: 'sender@inbox.example.com', + recipient: 'recipient@example.net', + terminal: true, + delivery: { status: 'bounced' }, + }, + metadata: { + accountId: 'account-1', + eventSubscriptionId: 'subscription-1', + eventSchemaVersion: 1, + eventTimestamp: '2026-08-01T13:03:00.000Z', + }, + }, + }) + ).outcome, + ).toBe('unmatched') +}) + +test('deleteEmailMessageById removes the derived provider index row', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = `idx-delete-${crypto.randomUUID()}` + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'Delete index', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-08-01T14:00:00.000Z', + }, + }) + + const deletion = await deleteEmailMessageById({ + db: env.APP_DB, + blobs: env.EMAIL_BLOBS, + messageId: message.id, + expectedUserId: userId, + }) + expect(deletion.messageFound).toBe(true) + expect( + await getOutboundProviderIndexRow({ + db: env.APP_DB, + providerMessageId, + }), + ).toBeNull() + expect( + await getOutboundEmailMessageByProviderMessageId({ + db: env.APP_DB, + providerMessageId, + }), + ).toBeNull() +}) + +test('parity report flags missing and mismatched index rows', async () => { + await ensureEmailTestSchema(env.APP_DB) + const userId = `idx-parity-${crypto.randomUUID()}` + const providerMessageId = `provider-${crypto.randomUUID()}` + const message = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'Parity', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-08-01T15:00:00.000Z', + }, + }) + await env.APP_DB.prepare( + `DELETE FROM email_outbound_provider_index WHERE message_id = ?`, + ) + .bind(message.id) + .run() + await env.APP_DB.prepare( + `INSERT INTO email_outbound_provider_index ( + provider, provider_message_id, user_id, message_id, inbox_id, + created_at, updated_at + ) VALUES (?, ?, ?, ?, NULL, ?, ?)`, + ) + .bind( + emailOutboundProviderCloudflare, + `orphan-${crypto.randomUUID()}`, + userId, + `missing-message-${crypto.randomUUID()}`, + '2026-08-01T15:00:00.000Z', + '2026-08-01T15:00:00.000Z', + ) + .run() + + const report = await loadOutboundProviderIndexParityReport({ + db: env.APP_DB, + userId, + }) + expect(report.parity).toBe(false) + expect(report.missingFromIndexCount).toBe(1) + expect(report.missingFromMessagesCount).toBe(1) + expect(report.sampleMismatches.map((row) => row.kind).sort()).toEqual([ + 'missing_from_index', + 'missing_from_messages', + ]) +}) + +test('sendOutboundEmail dual-writes the provider index on terminal success', async () => { + silenceIncidentalRuntimeWarnings() + await ensureEmailTestSchema(env.APP_DB) + const email = `idx-send-${crypto.randomUUID()}@example.com` + const userId = await seedVerifiedAccount(email) + + const result = await sendOutboundEmail({ + env: createBindingSendEnv(), + userId, + accountEmail: email, + recipientPolicy: 'self', + subject: 'Outbound index', + text: 'hello', + }) + expect(result.status).toBe('sent') + expect(result.providerMessageId).toEqual(expect.any(String)) + expect( + await getOutboundProviderIndexRow({ + db: env.APP_DB, + providerMessageId: result.providerMessageId!, + }), + ).toMatchObject({ + userId, + messageId: result.message.id, + provider: emailOutboundProviderCloudflare, + }) + expect( + await loadOutboundProviderIndexParityReport({ + db: env.APP_DB, + userId, + }), + ).toMatchObject({ parity: true, linkedMessageCount: 1, indexCount: 1 }) +}, 30_000) diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index 136e49e4ff..c03d1ca88c 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -751,6 +751,8 @@ export async function sendOutboundEmail( providerMessageId, error: null, sentAt: new Date().toISOString(), + userId: input.userId, + inboxId: message.inboxId, }) await insertEmailDeliveryEvent({ db: input.env.APP_DB, diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 10ada76802..19bbe924aa 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -4,6 +4,11 @@ import { utf8ByteLength, } from '@kody-internal/shared/backup-restore-safety.ts' import { parseJsonArray } from '@kody-internal/shared/json-parsing.ts' +import { + deleteOutboundProviderIndexByMessageId, + getOutboundProviderIndexRow, + upsertOutboundProviderIndex, +} from './outbound-provider-index.ts' import { emailClassificationValues, type EmailAttachmentRecord, @@ -804,7 +809,22 @@ export async function insertEmailMessage(input: { 'Inbound delivery storage lease was lost before message insert.', ) } - return mapMessageRow(row) + const message = mapMessageRow(row) + if ( + message.direction === 'outbound' && + message.providerMessageId != null && + message.providerMessageId !== '' + ) { + await upsertOutboundProviderIndex({ + db: input.db, + providerMessageId: message.providerMessageId, + userId: message.userId, + messageId: message.id, + inboxId: message.inboxId, + now: message.updatedAt, + }) + } + return message } export async function updateEmailMessageDelivery(input: { @@ -814,7 +834,14 @@ export async function updateEmailMessageDelivery(input: { providerMessageId?: string | null error?: string | null sentAt?: string | null + /** + * Owner fields for the derived provider reverse index dual-write. When + * omitted and a provider id is set, the message row is read after update. + */ + userId?: string + inboxId?: string | null }) { + const updatedAt = nowIso() await input.db .prepare( `UPDATE email_messages @@ -830,10 +857,44 @@ export async function updateEmailMessageDelivery(input: { input.providerMessageId ?? null, input.error ?? null, input.sentAt ?? null, - nowIso(), + updatedAt, input.messageId, ) .run() + + const providerMessageId = input.providerMessageId ?? null + if (providerMessageId == null || providerMessageId === '') { + await deleteOutboundProviderIndexByMessageId({ + db: input.db, + messageId: input.messageId, + }) + return + } + + let userId = input.userId + let inboxId = input.inboxId ?? null + if (userId == null) { + const row = await input.db + .prepare( + `SELECT user_id, inbox_id + FROM email_messages + WHERE id = ? + LIMIT 1`, + ) + .bind(input.messageId) + .first<{ user_id: string; inbox_id: string | null }>() + if (!row) return + userId = row.user_id + inboxId = row.inbox_id + } + await upsertOutboundProviderIndex({ + db: input.db, + providerMessageId, + userId, + messageId: input.messageId, + inboxId, + now: updatedAt, + }) } export async function getEmailMessageById(input: { @@ -888,27 +949,28 @@ export async function updateEmailMessageClassificationInD1(input: { return Number(result.meta.changes ?? 0) > 0 } +/** + * Resolve an outbound message from the derived provider reverse index, then + * load the owner-scoped authoritative `email_messages` row. Does not scan the + * full messages table by provider_message_id. + */ export async function getOutboundEmailMessageByProviderMessageId(input: { db: D1Database providerMessageId: string }) { - const result = await input.db - .prepare( - `SELECT * - FROM email_messages - WHERE direction = 'outbound' - AND provider_message_id = ? - LIMIT 2`, - ) - .bind(input.providerMessageId) - .all>() - const rows = result.results ?? [] - if (rows.length > 1) { - throw new Error( - `Multiple outbound email messages share provider id: ${input.providerMessageId}`, - ) - } - return rows[0] ? mapMessageRow(rows[0]) : null + const indexRow = await getOutboundProviderIndexRow({ + db: input.db, + providerMessageId: input.providerMessageId, + }) + if (!indexRow) return null + const message = await getEmailMessageById({ + db: input.db, + userId: indexRow.userId, + messageId: indexRow.messageId, + }) + if (!message || message.direction !== 'outbound') return null + if (message.providerMessageId !== input.providerMessageId) return null + return message } export async function getEmailMessageByMessageIdHeader(input: { @@ -1244,10 +1306,14 @@ export async function deleteEmailMessageById(input: { ] // Atomic batch: a partial delete (attachments gone, message left) would // lose the storage_key values needed to delete the R2 blobs on retry. + // The derived provider reverse index is cleared with the message row. await input.db.batch([ input.db .prepare(`DELETE FROM email_attachments WHERE message_id = ?`) .bind(input.messageId), + input.db + .prepare(`DELETE FROM email_outbound_provider_index WHERE message_id = ?`) + .bind(input.messageId), input.db .prepare(`DELETE FROM email_messages WHERE id = ?`) .bind(input.messageId), diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index 646320339b..19fea67aa7 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -760,6 +760,11 @@ export async function setEmailMessageClassification(input: { return true } +/** + * Record a Cloudflare outbound delivery lifecycle event. Resolution is + * index-first via `email_outbound_provider_index`, then an owner-scoped + * `email_messages` load — never a full-table provider_message_id scan. + */ export async function recordProviderEmailDeliveryEvent(input: { db: D1Database reportingEnv?: EmailReportingEnv diff --git a/packages/worker/src/email/system-email.ts b/packages/worker/src/email/system-email.ts index f373e17aff..6ada88bd4e 100644 --- a/packages/worker/src/email/system-email.ts +++ b/packages/worker/src/email/system-email.ts @@ -1,5 +1,8 @@ import { utcDayKey } from '@kody-internal/shared/date-keys.ts' +import { systemEmailOwnerId } from './email-owner.ts' +import { deleteOutboundProviderIndexByMessageIds } from './outbound-provider-index.ts' import { maxRawMimeBytes } from './parser.ts' +import { buildPlatformEmailAddress } from './platform-address.ts' import { createEmailInbox, createEmailInboxAddress, @@ -9,8 +12,6 @@ import { getEmailInboxById, getEmailInboxByName, } from './repo.ts' -import { buildPlatformEmailAddress } from './platform-address.ts' -import { systemEmailOwnerId } from './email-owner.ts' import { type EmailInboxAddressRecord, type EmailInboxRecord } from './types.ts' export { systemEmailOwnerId } @@ -323,6 +324,10 @@ async function deleteSystemEmailMessagesByIds(input: { ) .bind(...deletableIds) .run() + await deleteOutboundProviderIndexByMessageIds({ + db: input.db, + messageIds: deletableIds, + }) await input.db .prepare( `DELETE FROM email_messages diff --git a/packages/worker/src/email/test-schema.ts b/packages/worker/src/email/test-schema.ts index 7fbc7192bc..1b248447e3 100644 --- a/packages/worker/src/email/test-schema.ts +++ b/packages/worker/src/email/test-schema.ts @@ -10,6 +10,7 @@ export async function ensureEmailTestSchema(db: D1Database) { `DROP TABLE IF EXISTS email_sender_rules;`, `DROP TABLE IF EXISTS email_delivery_events;`, `DROP TABLE IF EXISTS email_attachments;`, + `DROP TABLE IF EXISTS email_outbound_provider_index;`, `DROP TABLE IF EXISTS email_messages;`, `DROP TABLE IF EXISTS email_threads;`, `DROP TABLE IF EXISTS email_inbox_addresses;`, @@ -169,6 +170,20 @@ WHERE provider = 'cloudflare-email-routing' ON email_messages(provider_message_id) WHERE direction = 'outbound' AND provider_message_id IS NOT NULL;`, + `CREATE TABLE IF NOT EXISTS email_outbound_provider_index ( + provider TEXT NOT NULL, + provider_message_id TEXT NOT NULL, + user_id TEXT NOT NULL, + message_id TEXT NOT NULL, + inbox_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (provider, provider_message_id) +);`, + `CREATE INDEX IF NOT EXISTS idx_email_outbound_provider_index_user_id +ON email_outbound_provider_index(user_id);`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_email_outbound_provider_index_message_id +ON email_outbound_provider_index(message_id);`, `CREATE UNIQUE INDEX IF NOT EXISTS idx_email_delivery_events_provider_event_id ON email_delivery_events(provider_event_id) WHERE provider_event_id IS NOT NULL;`, diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index d1e351a43f..12d28901c3 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -523,6 +523,10 @@ { "filename": "0127-drop-legacy-audit-events.sql", "sha256": "414990ca0d5bf7a1c8cf9d06cffbf46b5e6c3f7cf2dcbfccb00eeca0fa067d70" + }, + { + "filename": "0128-email-outbound-provider-index.sql", + "sha256": "84301bf6d3c533db15f94aaabde2a34d7ecb1fb196578666b6bfd3c2dffd3408" } ] } From 730c41cea7bca8dca90d5b485a5abce0e67acfba Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 04:53:36 +0000 Subject: [PATCH 44/62] fix(email): commit provider index atomically Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 20 +- .../admin/mailbox-maintenance.node.test.ts | 36 ++ .../worker/src/admin/mailbox-maintenance.ts | 14 + .../src/email/outbound-provider-index.ts | 402 ++++++------------ .../outbound-provider-index.workers.test.ts | 17 +- packages/worker/src/email/repo.ts | 122 ++---- .../admin-mailbox-maintenance.node.test.ts | 10 +- .../admin/admin-mailbox-maintenance.ts | 31 +- 8 files changed, 289 insertions(+), 363 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index ce81ea0763..ee830b98e7 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -1121,19 +1121,21 @@ below. `0128-email-outbound-provider-index.sql`), keyed by `(provider, provider_message_id)` with `user_id`, `message_id`, `inbox_id`, and created/updated timestamps (indexes on `user_id` and unique `message_id`). - `email_messages.provider_message_id` remains authoritative: outbound terminal - success / provider-id updates dual-write the index after the message row - write; `deleteEmailMessageById`, user/system retention, and account deletion - remove owner index rows. Account export treats the table as derived global - lookup (`includeInExport: false` / + `email_messages.provider_message_id` remains authoritative: outbound inserts + with a provider id and `updateEmailMessageDelivery` commit the message row + plus index sync in one `db.batch` (index owner/inbox fields come from the + authoritative message row, never caller input). `deleteEmailMessageById`, + user/system retention, and account deletion remove owner index rows. Account + export treats the table as derived global lookup (`includeInExport: false` / `derivedData.email_outbound_provider_index`) because authoritative outbound message rows are already exported. `recordProviderEmailDeliveryEvent` resolves index-first, then loads the owner-scoped message by `user_id`/`message_id` (no full-table provider scan). `system:email` rows are indexed when they carry a - provider id. Read helper `loadOutboundProviderIndexParityReport` compares - linked outbound messages vs index counts/missing/mismatched for production - evidence. This phase does not flip Mailbox write authority; contextless - provider-id reverse lookups must not enumerate per-user Mailbox objects. + provider id. Aggregate parity (`loadOutboundProviderIndexParityReport`; counts + only) is surfaced on `admin_mailbox_maintenance` + `status.outboundProviderIndex` for production verification. This phase does + not flip Mailbox write authority; contextless provider-id reverse lookups must + not enumerate per-user Mailbox objects. ### Inbound durability boundary (D1-authoritative dual-write) diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts index 56779f3ad3..bf6859ddfb 100644 --- a/packages/worker/src/admin/mailbox-maintenance.node.test.ts +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -96,8 +96,21 @@ function createMaintenanceDb() { CREATE TABLE email_messages ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, + direction TEXT NOT NULL DEFAULT 'inbound', + provider_message_id TEXT, + inbox_id TEXT, created_at TEXT NOT NULL DEFAULT '${freshCreatedAt}' ); + CREATE TABLE email_outbound_provider_index ( + provider TEXT NOT NULL, + provider_message_id TEXT NOT NULL, + user_id TEXT NOT NULL, + message_id TEXT NOT NULL, + inbox_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (provider, provider_message_id) + ); CREATE TABLE email_delivery_events ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, @@ -359,6 +372,19 @@ test('loadAdminMailboxMaintenanceStatus aggregates buckets without owner ids', a eventsCompletedAt: matchingSince, }) + sqlite.exec(` + INSERT INTO email_messages ( + id, user_id, direction, provider_message_id, created_at + ) VALUES + ('linked-1', 'user-linked', 'outbound', 'prov-1', '${freshCreatedAt}'); + INSERT INTO email_outbound_provider_index ( + provider, provider_message_id, user_id, message_id, inbox_id, + created_at, updated_at + ) VALUES + ('cloudflare-email', 'prov-1', 'user-linked', 'linked-1', NULL, + '${freshCreatedAt}', '${freshCreatedAt}'); + `) + const status = await loadAdminMailboxMaintenanceStatus({ db, now }) expect(status).toMatchObject({ generatedAt: now.toISOString(), @@ -368,7 +394,17 @@ test('loadAdminMailboxMaintenanceStatus aggregates buckets without owner ids', a error: 1, incomplete: 1, eligible: 1, + outboundProviderIndex: { + linkedMessageCount: 1, + indexCount: 1, + missingFromIndexCount: 0, + missingFromMessagesCount: 0, + mismatchedCount: 0, + parity: true, + }, }) + expect(status).not.toHaveProperty('outboundProviderIndex.sampleMismatches') + expect(JSON.stringify(status.outboundProviderIndex)).not.toMatch(/@|prov-1/u) }) test('runAdminMailboxMaintenanceReconcile clamps batch_size and returns status', async () => { diff --git a/packages/worker/src/admin/mailbox-maintenance.ts b/packages/worker/src/admin/mailbox-maintenance.ts index b191edd6dc..940c1242bf 100644 --- a/packages/worker/src/admin/mailbox-maintenance.ts +++ b/packages/worker/src/admin/mailbox-maintenance.ts @@ -17,6 +17,10 @@ import { mailboxReadCutoverSoakMs, } from '#worker/email/mailbox-read-cutover.ts' import { type MailboxCountResult } from '#worker/email/mailbox-types.ts' +import { + loadOutboundProviderIndexParityReport, + type OutboundProviderIndexParityReport, +} from '#worker/email/outbound-provider-index.ts' import { deleteEmailMessageById, getEmailMessageById, @@ -55,6 +59,11 @@ export type AdminMailboxMaintenanceStatus = { newestCheckedAt: string | null /** Earliest `matching_since + soak` among matching owners (null if none). */ earliestCutoverAt: string | null + /** + * Fleet-wide aggregate D1 outbound provider reverse-index parity. Counts + * only — no owner ids, message ids, or email content. + */ + outboundProviderIndex: OutboundProviderIndexParityReport } export type AdminMailboxMaintenanceRetentionD1Metrics = { @@ -436,6 +445,10 @@ export async function loadAdminMailboxMaintenanceStatus(input: { } } + const outboundProviderIndex = await loadOutboundProviderIndexParityReport({ + db: input.db, + }) + return { generatedAt, trackedOwners: Number(row?.trackedOwners ?? 0) || 0, @@ -449,6 +462,7 @@ export async function loadAdminMailboxMaintenanceStatus(input: { oldestCheckedAt: row?.oldestCheckedAt ?? null, newestCheckedAt: row?.newestCheckedAt ?? null, earliestCutoverAt, + outboundProviderIndex, } } diff --git a/packages/worker/src/email/outbound-provider-index.ts b/packages/worker/src/email/outbound-provider-index.ts index b3f9f8d7b3..d5d679b61b 100644 --- a/packages/worker/src/email/outbound-provider-index.ts +++ b/packages/worker/src/email/outbound-provider-index.ts @@ -17,6 +17,15 @@ export type EmailOutboundProviderIndexRow = { updatedAt: string } +export type OutboundProviderIndexParityReport = { + linkedMessageCount: number + indexCount: number + missingFromIndexCount: number + missingFromMessagesCount: number + mismatchedCount: number + parity: boolean +} + function mapIndexRow( row: Record, ): EmailOutboundProviderIndexRow { @@ -51,51 +60,68 @@ export async function getOutboundProviderIndexRow(input: { } /** - * Upsert the derived index for one outbound message. Callers must write the - * authoritative `email_messages` row first. Replaces any prior mapping for the - * same `message_id` when the provider id changes. + * Prepared statements that sync the derived index from the authoritative + * `email_messages` row. Callers must include these in the same `db.batch` as + * the message INSERT/UPDATE so message + index commit atomically. Owner and + * inbox fields always come from the message row — never from caller input. */ -export async function upsertOutboundProviderIndex(input: { +export function prepareOutboundProviderIndexSyncStatements(input: { db: D1Database - provider?: string - providerMessageId: string - userId: string messageId: string - inboxId?: string | null - now?: string -}): Promise { + /** + * Target provider id after the accompanying message mutation. Null/empty + * clears every index row for `messageId`. + */ + providerMessageId: string | null + provider?: string + now: string +}): Array { const provider = input.provider ?? emailOutboundProviderCloudflare - const now = input.now ?? new Date().toISOString() - await input.db.batch([ + const providerMessageId = input.providerMessageId + if (providerMessageId == null || providerMessageId === '') { + return [ + input.db + .prepare( + `DELETE FROM email_outbound_provider_index + WHERE message_id = ?`, + ) + .bind(input.messageId), + ] + } + return [ input.db .prepare( `DELETE FROM email_outbound_provider_index WHERE message_id = ? AND NOT (provider = ? AND provider_message_id = ?)`, ) - .bind(input.messageId, provider, input.providerMessageId), + .bind(input.messageId, provider, providerMessageId), input.db .prepare( `INSERT INTO email_outbound_provider_index ( provider, provider_message_id, user_id, message_id, inbox_id, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?) + ) + SELECT + ?, + message.provider_message_id, + message.user_id, + message.id, + message.inbox_id, + ?, + ? + FROM email_messages AS message + WHERE message.id = ? + AND message.direction = 'outbound' + AND message.provider_message_id = ? ON CONFLICT(provider, provider_message_id) DO UPDATE SET user_id = excluded.user_id, message_id = excluded.message_id, inbox_id = excluded.inbox_id, updated_at = excluded.updated_at`, ) - .bind( - provider, - input.providerMessageId, - input.userId, - input.messageId, - input.inboxId ?? null, - now, - now, - ), - ]) + .bind(provider, input.now, input.now, input.messageId, providerMessageId), + ] } export async function deleteOutboundProviderIndexByMessageId(input: { @@ -134,265 +160,111 @@ export async function deleteOutboundProviderIndexByMessageIds(input: { return deleted } -export type OutboundProviderIndexParityMismatch = { - kind: 'missing_from_index' | 'missing_from_messages' | 'mismatched' - provider: string - providerMessageId: string - userId: string - messageId: string - detail: string -} - -export type OutboundProviderIndexParityReport = { - linkedMessageCount: number - indexCount: number - missingFromIndexCount: number - missingFromMessagesCount: number - mismatchedCount: number - sampleMismatches: Array - parity: boolean -} - /** - * Read-only compare of outbound messages that carry a provider id versus the - * derived reverse index. Optional `userId` scopes the report to one owner - * (including `system:email`). + * Read-only aggregate compare of outbound messages that carry a provider id + * versus the derived reverse index. Optional `userId` scopes the report + * (including `system:email`). No message content is returned. */ export async function loadOutboundProviderIndexParityReport(input: { db: D1Database userId?: string - sampleLimit?: number }): Promise { - const sampleLimit = input.sampleLimit ?? 20 - const userFilter = input.userId == null ? '' : ' AND user_id = ?' - const userBindings = input.userId == null ? [] : [input.userId] - - const linkedMessageCount = Number( - ( - await input.db - .prepare( - `SELECT COUNT(*) AS count - FROM email_messages - WHERE direction = 'outbound' - AND provider_message_id IS NOT NULL${userFilter}`, - ) - .bind(...userBindings) - .first<{ count: number }>() - )?.count ?? 0, - ) - const indexCount = Number( - ( - await input.db - .prepare( - `SELECT COUNT(*) AS count - FROM email_outbound_provider_index - WHERE 1 = 1${userFilter}`, - ) - .bind(...userBindings) - .first<{ count: number }>() - )?.count ?? 0, - ) - - const missingFromIndexRows = await input.db - .prepare( - `SELECT - message.id AS message_id, - message.user_id AS user_id, - message.provider_message_id AS provider_message_id - FROM email_messages AS message - WHERE message.direction = 'outbound' - AND message.provider_message_id IS NOT NULL${ - input.userId == null ? '' : ' AND message.user_id = ?' - } - AND NOT EXISTS ( - SELECT 1 - FROM email_outbound_provider_index AS idx - WHERE idx.provider = ? - AND idx.provider_message_id = message.provider_message_id - AND idx.message_id = message.id - AND idx.user_id = message.user_id - ) - ORDER BY message.created_at ASC, message.id ASC - LIMIT ?`, - ) - .bind( - ...(input.userId == null ? [] : [input.userId]), - emailOutboundProviderCloudflare, - sampleLimit, - ) - .all<{ - message_id: string - user_id: string - provider_message_id: string - }>() - - const missingFromMessagesRows = await input.db - .prepare( - `SELECT - idx.provider AS provider, - idx.provider_message_id AS provider_message_id, - idx.user_id AS user_id, - idx.message_id AS message_id - FROM email_outbound_provider_index AS idx - WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} - AND NOT EXISTS ( - SELECT 1 - FROM email_messages AS message - WHERE message.id = idx.message_id - AND message.user_id = idx.user_id - AND message.direction = 'outbound' - AND message.provider_message_id = idx.provider_message_id - ) - ORDER BY idx.created_at ASC, idx.message_id ASC - LIMIT ?`, - ) - .bind(...(input.userId == null ? [] : [input.userId]), sampleLimit) - .all<{ - provider: string - provider_message_id: string - user_id: string - message_id: string - }>() - - const mismatchedRows = await input.db + const userId = input.userId ?? null + const row = await input.db .prepare( - `SELECT - idx.provider AS provider, - idx.provider_message_id AS provider_message_id, - idx.user_id AS index_user_id, - idx.message_id AS message_id, - message.user_id AS message_user_id, - message.provider_message_id AS message_provider_message_id, - message.inbox_id AS message_inbox_id, - idx.inbox_id AS index_inbox_id - FROM email_outbound_provider_index AS idx - JOIN email_messages AS message - ON message.id = idx.message_id - WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} - AND ( - message.user_id != idx.user_id - OR message.direction != 'outbound' - OR message.provider_message_id IS NULL - OR message.provider_message_id != idx.provider_message_id - OR IFNULL(message.inbox_id, '') != IFNULL(idx.inbox_id, '') - ) - ORDER BY idx.created_at ASC, idx.message_id ASC - LIMIT ?`, - ) - .bind(...(input.userId == null ? [] : [input.userId]), sampleLimit) - .all<{ - provider: string - provider_message_id: string - index_user_id: string - message_id: string - message_user_id: string - message_provider_message_id: string | null - message_inbox_id: string | null - index_inbox_id: string | null - }>() - - const missingFromIndexCount = Number( - ( - await input.db - .prepare( - `SELECT COUNT(*) AS count + `WITH + linked_messages AS ( + SELECT + message.id AS message_id, + message.user_id AS user_id, + message.inbox_id AS inbox_id, + message.provider_message_id AS provider_message_id FROM email_messages AS message WHERE message.direction = 'outbound' - AND message.provider_message_id IS NOT NULL${ - input.userId == null ? '' : ' AND message.user_id = ?' - } - AND NOT EXISTS ( - SELECT 1 - FROM email_outbound_provider_index AS idx - WHERE idx.provider = ? - AND idx.provider_message_id = message.provider_message_id - AND idx.message_id = message.id - AND idx.user_id = message.user_id - )`, - ) - .bind( - ...(input.userId == null ? [] : [input.userId]), - emailOutboundProviderCloudflare, - ) - .first<{ count: number }>() - )?.count ?? 0, - ) - const missingFromMessagesCount = Number( - ( - await input.db - .prepare( - `SELECT COUNT(*) AS count - FROM email_outbound_provider_index AS idx - WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} - AND NOT EXISTS ( - SELECT 1 - FROM email_messages AS message - WHERE message.id = idx.message_id - AND message.user_id = idx.user_id - AND message.direction = 'outbound' - AND message.provider_message_id = idx.provider_message_id - )`, - ) - .bind(...(input.userId == null ? [] : [input.userId])) - .first<{ count: number }>() - )?.count ?? 0, - ) - const mismatchedCount = Number( - ( - await input.db - .prepare( - `SELECT COUNT(*) AS count + AND message.provider_message_id IS NOT NULL + AND (?1 IS NULL OR message.user_id = ?1) + ), + index_rows AS ( + SELECT + idx.provider AS provider, + idx.provider_message_id AS provider_message_id, + idx.user_id AS user_id, + idx.message_id AS message_id, + idx.inbox_id AS inbox_id FROM email_outbound_provider_index AS idx - JOIN email_messages AS message - ON message.id = idx.message_id - WHERE 1 = 1${input.userId == null ? '' : ' AND idx.user_id = ?'} - AND ( - message.user_id != idx.user_id - OR message.direction != 'outbound' - OR message.provider_message_id IS NULL - OR message.provider_message_id != idx.provider_message_id - OR IFNULL(message.inbox_id, '') != IFNULL(idx.inbox_id, '') - )`, + WHERE ?1 IS NULL OR idx.user_id = ?1 + ), + classified AS ( + SELECT + (SELECT COUNT(*) FROM linked_messages) AS linked_message_count, + (SELECT COUNT(*) FROM index_rows) AS index_count, + ( + SELECT COUNT(*) + FROM linked_messages AS message + WHERE NOT EXISTS ( + SELECT 1 + FROM index_rows AS idx + WHERE idx.provider = ?2 + AND idx.provider_message_id = message.provider_message_id + AND idx.message_id = message.message_id + AND idx.user_id = message.user_id + AND IFNULL(idx.inbox_id, '') = IFNULL(message.inbox_id, '') + ) + ) AS missing_from_index_count, + ( + SELECT COUNT(*) + FROM index_rows AS idx + WHERE NOT EXISTS ( + SELECT 1 + FROM linked_messages AS message + WHERE message.message_id = idx.message_id + AND message.user_id = idx.user_id + AND message.provider_message_id = idx.provider_message_id + ) + ) AS missing_from_messages_count, + ( + SELECT COUNT(*) + FROM index_rows AS idx + JOIN email_messages AS message + ON message.id = idx.message_id + WHERE (?1 IS NULL OR idx.user_id = ?1) + AND ( + message.user_id != idx.user_id + OR message.direction != 'outbound' + OR message.provider_message_id IS NULL + OR message.provider_message_id != idx.provider_message_id + OR IFNULL(message.inbox_id, '') != IFNULL(idx.inbox_id, '') + ) + ) AS mismatched_count ) - .bind(...(input.userId == null ? [] : [input.userId])) - .first<{ count: number }>() - )?.count ?? 0, - ) - - const sampleMismatches: Array = [ - ...(missingFromIndexRows.results ?? []).map((row) => ({ - kind: 'missing_from_index' as const, - provider: emailOutboundProviderCloudflare, - providerMessageId: row.provider_message_id, - userId: row.user_id, - messageId: row.message_id, - detail: 'Linked outbound message has no matching index row.', - })), - ...(missingFromMessagesRows.results ?? []).map((row) => ({ - kind: 'missing_from_messages' as const, - provider: row.provider, - providerMessageId: row.provider_message_id, - userId: row.user_id, - messageId: row.message_id, - detail: 'Index row has no matching linked outbound message.', - })), - ...(mismatchedRows.results ?? []).map((row) => ({ - kind: 'mismatched' as const, - provider: row.provider, - providerMessageId: row.provider_message_id, - userId: row.index_user_id, - messageId: row.message_id, - detail: `Index/message fields diverge (message user=${row.message_user_id}, provider_message_id=${row.message_provider_message_id ?? 'null'}, inbox=${row.message_inbox_id ?? 'null'} vs index inbox=${row.index_inbox_id ?? 'null'}).`, - })), - ].slice(0, sampleLimit) + SELECT + linked_message_count, + index_count, + missing_from_index_count, + missing_from_messages_count, + mismatched_count + FROM classified`, + ) + .bind(userId, emailOutboundProviderCloudflare) + .first<{ + linked_message_count: number + index_count: number + missing_from_index_count: number + missing_from_messages_count: number + mismatched_count: number + }>() + const linkedMessageCount = Number(row?.linked_message_count ?? 0) + const indexCount = Number(row?.index_count ?? 0) + const missingFromIndexCount = Number(row?.missing_from_index_count ?? 0) + const missingFromMessagesCount = Number(row?.missing_from_messages_count ?? 0) + const mismatchedCount = Number(row?.mismatched_count ?? 0) return { linkedMessageCount, indexCount, missingFromIndexCount, missingFromMessagesCount, mismatchedCount, - sampleMismatches, parity: linkedMessageCount === indexCount && missingFromIndexCount === 0 && diff --git a/packages/worker/src/email/outbound-provider-index.workers.test.ts b/packages/worker/src/email/outbound-provider-index.workers.test.ts index 2479a775b0..20339f18a3 100644 --- a/packages/worker/src/email/outbound-provider-index.workers.test.ts +++ b/packages/worker/src/email/outbound-provider-index.workers.test.ts @@ -90,8 +90,6 @@ test('insert and delivery update dual-write the outbound provider index', async messageId: message.id, status: 'sent', providerMessageId: rotatedProviderMessageId, - userId, - inboxId: null, sentAt: '2026-08-01T12:01:00.000Z', }) @@ -336,13 +334,14 @@ test('parity report flags missing and mismatched index rows', async () => { db: env.APP_DB, userId, }) - expect(report.parity).toBe(false) - expect(report.missingFromIndexCount).toBe(1) - expect(report.missingFromMessagesCount).toBe(1) - expect(report.sampleMismatches.map((row) => row.kind).sort()).toEqual([ - 'missing_from_index', - 'missing_from_messages', - ]) + expect(report).toMatchObject({ + parity: false, + linkedMessageCount: 1, + indexCount: 1, + missingFromIndexCount: 1, + missingFromMessagesCount: 1, + }) + expect(report).not.toHaveProperty('sampleMismatches') }) test('sendOutboundEmail dual-writes the provider index on terminal success', async () => { diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 19bbe924aa..9e3847dcf3 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -5,9 +5,8 @@ import { } from '@kody-internal/shared/backup-restore-safety.ts' import { parseJsonArray } from '@kody-internal/shared/json-parsing.ts' import { - deleteOutboundProviderIndexByMessageId, getOutboundProviderIndexRow, - upsertOutboundProviderIndex, + prepareOutboundProviderIndexSyncStatements, } from './outbound-provider-index.ts' import { emailClassificationValues, @@ -745,7 +744,7 @@ export async function insertEmailMessage(input: { updated_at: timestamp, } const fence = input.inboundDeliveryFence - const result = await input.db + const insertStatement = input.db .prepare( `INSERT INTO email_messages ( id, direction, user_id, inbox_id, thread_id, sender_identity_id, @@ -803,28 +802,29 @@ export async function insertEmailMessage(input: { row.updated_at, ...(fence ? [fence.deliveryId, fence.userId, fence.storageLease] : []), ) - .run() - if (fence && Number(result.meta.changes ?? 0) === 0) { + const shouldIndexProvider = + row.direction === 'outbound' && + row.provider_message_id != null && + row.provider_message_id !== '' + const insertResult = shouldIndexProvider + ? ( + await input.db.batch([ + insertStatement, + ...prepareOutboundProviderIndexSyncStatements({ + db: input.db, + messageId: row.id, + providerMessageId: row.provider_message_id, + now: row.updated_at, + }), + ]) + )[0] + : await insertStatement.run() + if (fence && Number(insertResult?.meta.changes ?? 0) === 0) { throw new Error( 'Inbound delivery storage lease was lost before message insert.', ) } - const message = mapMessageRow(row) - if ( - message.direction === 'outbound' && - message.providerMessageId != null && - message.providerMessageId !== '' - ) { - await upsertOutboundProviderIndex({ - db: input.db, - providerMessageId: message.providerMessageId, - userId: message.userId, - messageId: message.id, - inboxId: message.inboxId, - now: message.updatedAt, - }) - } - return message + return mapMessageRow(row) } export async function updateEmailMessageDelivery(input: { @@ -834,67 +834,35 @@ export async function updateEmailMessageDelivery(input: { providerMessageId?: string | null error?: string | null sentAt?: string | null - /** - * Owner fields for the derived provider reverse index dual-write. When - * omitted and a provider id is set, the message row is read after update. - */ - userId?: string - inboxId?: string | null }) { const updatedAt = nowIso() - await input.db - .prepare( - `UPDATE email_messages - SET processing_status = ?, - provider_message_id = ?, - error = ?, - sent_at = ?, - updated_at = ? - WHERE id = ?`, - ) - .bind( - input.status, - input.providerMessageId ?? null, - input.error ?? null, - input.sentAt ?? null, - updatedAt, - input.messageId, - ) - .run() - const providerMessageId = input.providerMessageId ?? null - if (providerMessageId == null || providerMessageId === '') { - await deleteOutboundProviderIndexByMessageId({ - db: input.db, - messageId: input.messageId, - }) - return - } - - let userId = input.userId - let inboxId = input.inboxId ?? null - if (userId == null) { - const row = await input.db + await input.db.batch([ + input.db .prepare( - `SELECT user_id, inbox_id - FROM email_messages - WHERE id = ? - LIMIT 1`, + `UPDATE email_messages + SET processing_status = ?, + provider_message_id = ?, + error = ?, + sent_at = ?, + updated_at = ? + WHERE id = ?`, ) - .bind(input.messageId) - .first<{ user_id: string; inbox_id: string | null }>() - if (!row) return - userId = row.user_id - inboxId = row.inbox_id - } - await upsertOutboundProviderIndex({ - db: input.db, - providerMessageId, - userId, - messageId: input.messageId, - inboxId, - now: updatedAt, - }) + .bind( + input.status, + providerMessageId, + input.error ?? null, + input.sentAt ?? null, + updatedAt, + input.messageId, + ), + ...prepareOutboundProviderIndexSyncStatements({ + db: input.db, + messageId: input.messageId, + providerMessageId, + now: updatedAt, + }), + ]) } export async function getEmailMessageById(input: { diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts index 35b29dff04..36f3a5719f 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.node.test.ts @@ -60,6 +60,14 @@ const emptyStatus = { oldestCheckedAt: '2026-07-31T00:00:00.000Z', newestCheckedAt: '2026-08-01T11:00:00.000Z', earliestCutoverAt: '2026-07-31T00:00:00.000Z', + outboundProviderIndex: { + linkedMessageCount: 0, + indexCount: 0, + missingFromIndexCount: 0, + missingFromMessagesCount: 0, + mismatchedCount: 0, + parity: true, + }, } const emptyRetentionResult = { @@ -183,7 +191,7 @@ test('admin_mailbox_maintenance status is audited and aggregate-only', async () expect.objectContaining({ action: 'admin_mailbox_maintenance', result: 'success', - reason: 'action=status;tracked=2', + reason: 'action=status;tracked=2;provider_index_parity=1', }), ) }) diff --git a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts index 46587a1b80..ce91dc019d 100644 --- a/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts +++ b/packages/worker/src/mcp/capabilities/admin/admin-mailbox-maintenance.ts @@ -44,6 +44,29 @@ const mailboxCountSchema = z.object({ deliveryEvents: z.number().int().nonnegative(), }) +const outboundProviderIndexParitySchema = z.object({ + linkedMessageCount: z + .number() + .int() + .nonnegative() + .describe( + 'Outbound email_messages rows with a non-null provider_message_id.', + ), + indexCount: z + .number() + .int() + .nonnegative() + .describe('Rows in email_outbound_provider_index.'), + missingFromIndexCount: z.number().int().nonnegative(), + missingFromMessagesCount: z.number().int().nonnegative(), + mismatchedCount: z.number().int().nonnegative(), + parity: z + .boolean() + .describe( + 'True when linked and index counts match and missing/mismatched counts are zero.', + ), +}) + const statusSchema = z.object({ generatedAt: z.string(), trackedOwners: z.number().int().nonnegative(), @@ -68,6 +91,9 @@ const statusSchema = z.object({ .describe( 'Earliest matching_since + soak duration among matching owners, or null.', ), + outboundProviderIndex: outboundProviderIndexParitySchema.describe( + 'Fleet-wide aggregate D1 outbound provider reverse-index parity (counts only; no owner/message content).', + ), }) const reconcileMetricsSchema = z.object({ @@ -214,12 +240,13 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( ...adminMutationCapabilityAccess, name: 'admin_mailbox_maintenance', description: - 'Admin-only Mailbox parity/retention/delete maintenance: aggregate status (no email content), bounded reconcileMailboxParity, keyset-paged natural retention (D1 authoritative prune, skip DO while owner still has expired D1 rows, then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget), or owner-scoped delete_message (getEmailMessageById ownership check, deleteEmailMessageById with expectedUserId fence, post-delete D1/R2 head over exact captured blob keys). Never accepts arbitrary cutoffs or seed data. Audited.', + 'Admin-only Mailbox parity/retention/delete maintenance: aggregate status (no email content) including fleet outbound provider-index parity counts, bounded reconcileMailboxParity, keyset-paged natural retention (D1 authoritative prune, skip DO while owner still has expired D1 rows, then Mailbox.runRetentionNow; limit≤20; concurrency≤4; ~10s budget), or owner-scoped delete_message (getEmailMessageById ownership check, deleteEmailMessageById with expectedUserId fence, post-delete D1/R2 head over exact captured blob keys). Never accepts arbitrary cutoffs or seed data. Audited.', keywords: [ 'admin', 'mailbox', 'maintenance', 'parity', + 'provider-index', 'reconcile', 'retention', 'delete', @@ -295,7 +322,7 @@ export const adminMailboxMaintenanceCapability = defineDomainCapability( successReason: (result) => { switch (result.action) { case 'status': - return `action=status;tracked=${result.status.trackedOwners}` + return `action=status;tracked=${result.status.trackedOwners};provider_index_parity=${result.status.outboundProviderIndex.parity ? 1 : 0}` case 'reconcile': return `action=reconcile;scanned=${result.metrics.scanned};matched=${result.metrics.matched}` case 'retention': From 8f95d8e736bb2c9d871aac6cc735139f233f2173 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 05:10:50 +0000 Subject: [PATCH 45/62] fix(email): preserve accepted provider sends Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 28 +- .../0128-email-outbound-provider-index.sql | 4 +- packages/worker/src/account/data-targets.ts | 2 +- .../admin/mailbox-maintenance.node.test.ts | 8 +- .../worker/src/app/retention.node.test.ts | 4 +- packages/worker/src/app/retention.ts | 6 +- ...ound-provider-index-atomicity.node.test.ts | 290 ++++++++++++++++++ ...ound-provider-index-migration.node.test.ts | 41 ++- .../outbound-provider-index.workers.test.ts | 105 ++++++- packages/worker/src/email/outbound.ts | 224 ++++++++------ packages/worker/src/email/repo.ts | 5 +- packages/worker/src/email/system-email.ts | 6 +- packages/worker/src/email/test-schema.ts | 3 +- tools/migration-ledger.json | 2 +- 14 files changed, 606 insertions(+), 122 deletions(-) create mode 100644 packages/worker/src/email/outbound-provider-index-atomicity.node.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index ee830b98e7..83e15056ce 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -1124,18 +1124,22 @@ below. `email_messages.provider_message_id` remains authoritative: outbound inserts with a provider id and `updateEmailMessageDelivery` commit the message row plus index sync in one `db.batch` (index owner/inbox fields come from the - authoritative message row, never caller input). `deleteEmailMessageById`, - user/system retention, and account deletion remove owner index rows. Account - export treats the table as derived global lookup (`includeInExport: false` / - `derivedData.email_outbound_provider_index`) because authoritative outbound - message rows are already exported. `recordProviderEmailDeliveryEvent` resolves - index-first, then loads the owner-scoped message by `user_id`/`message_id` (no - full-table provider scan). `system:email` rows are indexed when they carry a - provider id. Aggregate parity (`loadOutboundProviderIndexParityReport`; counts - only) is surfaced on `admin_mailbox_maintenance` - `status.outboundProviderIndex` for production verification. This phase does - not flip Mailbox write authority; contextless provider-id reverse lookups must - not enumerate per-user Mailbox objects. + authoritative message row, never caller input). `message_id` references + `email_messages(id)` with `ON DELETE CASCADE`, so message deletes clear index + rows; account deletion still inventories/deletes by `user_id` for coverage. + Outbound send separates provider acceptance from terminal D1/index + persistence: once the provider returns a `providerMessageId`, persistence uses + bounded D1 retries and must not mark the message `failed`, clear the id, or + resend. Account export treats the table as derived global lookup + (`includeInExport: false` / `derivedData.email_outbound_provider_index`) + because authoritative outbound message rows are already exported. + `recordProviderEmailDeliveryEvent` resolves index-first, then loads the + owner-scoped message by `user_id`/`message_id` (no full-table provider scan). + `system:email` rows are indexed when they carry a provider id. Aggregate + parity (`loadOutboundProviderIndexParityReport`; counts only) is surfaced on + `admin_mailbox_maintenance` `status.outboundProviderIndex` for production + verification. This phase does not flip Mailbox write authority; contextless + provider-id reverse lookups must not enumerate per-user Mailbox objects. ### Inbound durability boundary (D1-authoritative dual-write) diff --git a/packages/worker/migrations/0128-email-outbound-provider-index.sql b/packages/worker/migrations/0128-email-outbound-provider-index.sql index 2683060ba1..2cae5850ce 100644 --- a/packages/worker/migrations/0128-email-outbound-provider-index.sql +++ b/packages/worker/migrations/0128-email-outbound-provider-index.sql @@ -3,6 +3,7 @@ -- derived lookup so contextless delivery webhooks can resolve the owning user -- without a full-table provider_message_id scan (and without enumerating -- per-user Mailbox objects). Backfill includes system:email when present. +-- message_id FK cascades with authoritative message deletes. CREATE TABLE email_outbound_provider_index ( provider TEXT NOT NULL, @@ -12,7 +13,8 @@ CREATE TABLE email_outbound_provider_index ( inbox_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - PRIMARY KEY (provider, provider_message_id) + PRIMARY KEY (provider, provider_message_id), + FOREIGN KEY (message_id) REFERENCES email_messages(id) ON DELETE CASCADE ); CREATE INDEX idx_email_outbound_provider_index_user_id diff --git a/packages/worker/src/account/data-targets.ts b/packages/worker/src/account/data-targets.ts index 03c412d5ed..b7a68442ed 100644 --- a/packages/worker/src/account/data-targets.ts +++ b/packages/worker/src/account/data-targets.ts @@ -246,7 +246,7 @@ export const accountUserDataTargets: ReadonlyArray = [ includeInExport: false, surface: 'email_outbound_provider_index', reason: - 'Derived global provider→owner reverse lookup rebuilt from outbound email_messages.provider_message_id. Account deletion clears owner rows; export already includes the authoritative message rows and intentionally omits this lookup table.', + 'Derived global provider→owner reverse lookup rebuilt from outbound email_messages.provider_message_id. Message deletes cascade index rows via FK; account deletion still lists this table for inventory coverage and explicit owner cleanup. Export already includes the authoritative message rows and omits this lookup table.', }, { kind: 'user_id', table: 'email_messages' }, { kind: 'user_id', table: 'email_threads' }, diff --git a/packages/worker/src/admin/mailbox-maintenance.node.test.ts b/packages/worker/src/admin/mailbox-maintenance.node.test.ts index bf6859ddfb..de584d8fca 100644 --- a/packages/worker/src/admin/mailbox-maintenance.node.test.ts +++ b/packages/worker/src/admin/mailbox-maintenance.node.test.ts @@ -109,7 +109,8 @@ function createMaintenanceDb() { inbox_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - PRIMARY KEY (provider, provider_message_id) + PRIMARY KEY (provider, provider_message_id), + FOREIGN KEY (message_id) REFERENCES email_messages(id) ON DELETE CASCADE ); CREATE TABLE email_delivery_events ( id TEXT PRIMARY KEY, @@ -117,6 +118,7 @@ function createMaintenanceDb() { created_at TEXT NOT NULL DEFAULT '${freshCreatedAt}' ); `) + sqlite.exec('PRAGMA foreign_keys = ON') return { sqlite, db: createD1FromSqlite(sqlite) } } @@ -256,7 +258,8 @@ function createDeleteMessageDb() { inbox_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - PRIMARY KEY (provider, provider_message_id) + PRIMARY KEY (provider, provider_message_id), + FOREIGN KEY (message_id) REFERENCES email_messages(id) ON DELETE CASCADE ); CREATE TABLE email_attachments ( id TEXT PRIMARY KEY, @@ -269,6 +272,7 @@ function createDeleteMessageDb() { created_at TEXT NOT NULL ); `) + sqlite.exec('PRAGMA foreign_keys = ON') return { sqlite, db: createD1FromSqlite(sqlite) } } diff --git a/packages/worker/src/app/retention.node.test.ts b/packages/worker/src/app/retention.node.test.ts index e0fecdbca5..ed911e7561 100644 --- a/packages/worker/src/app/retention.node.test.ts +++ b/packages/worker/src/app/retention.node.test.ts @@ -96,6 +96,7 @@ function createD1FromSqlite( function createRetentionDb() { const sqlite = new DatabaseSync(':memory:') + sqlite.exec('PRAGMA foreign_keys = ON') sqlite.exec(` CREATE TABLE workflow_runs ( id TEXT PRIMARY KEY, @@ -197,7 +198,8 @@ function createRetentionDb() { inbox_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - PRIMARY KEY (provider, provider_message_id) + PRIMARY KEY (provider, provider_message_id), + FOREIGN KEY (message_id) REFERENCES email_messages(id) ON DELETE CASCADE ); CREATE TABLE email_attachments ( id TEXT PRIMARY KEY, diff --git a/packages/worker/src/app/retention.ts b/packages/worker/src/app/retention.ts index a97c0c0e37..048a1122ed 100644 --- a/packages/worker/src/app/retention.ts +++ b/packages/worker/src/app/retention.ts @@ -1,5 +1,4 @@ import { accountRetentionDispositions } from '#app/account-retention-dispositions.ts' -import { deleteOutboundProviderIndexByMessageIds } from '#worker/email/outbound-provider-index.ts' import { emailRawMimeKey } from '#worker/email/repo.ts' import { systemEmailOwnerId } from '#worker/email/system-email.ts' import { runD1WithRetry } from '#worker/d1-retry.ts' @@ -712,10 +711,7 @@ export async function pruneUserEmailMessagesForRetention(input: { idColumn: 'message_id', ids: messageIds, }) - await deleteOutboundProviderIndexByMessageIds({ - db: input.db, - messageIds: messageIds.map(String), - }) + // Provider-index rows cascade via FK ON DELETE CASCADE from email_messages. result.deletedMessages = await deleteByIds({ db: input.db, table: 'email_messages', diff --git a/packages/worker/src/email/outbound-provider-index-atomicity.node.test.ts b/packages/worker/src/email/outbound-provider-index-atomicity.node.test.ts new file mode 100644 index 0000000000..bdb619e479 --- /dev/null +++ b/packages/worker/src/email/outbound-provider-index-atomicity.node.test.ts @@ -0,0 +1,290 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { emailOutboundProviderCloudflare } from './outbound-provider-index.ts' +import { + getEmailMessageById, + getOutboundEmailMessageByProviderMessageId, + insertEmailMessage, + updateEmailMessageDelivery, +} from './repo.ts' + +function createEmailDb() { + const sqlite = new DatabaseSync(':memory:') + sqlite.exec('PRAGMA foreign_keys = ON') + sqlite.exec(` + CREATE TABLE email_messages ( + id TEXT PRIMARY KEY, + direction TEXT NOT NULL CHECK (direction IN ('inbound', 'outbound')), + user_id TEXT NOT NULL, + inbox_id TEXT, + thread_id TEXT, + sender_identity_id TEXT, + from_address TEXT NOT NULL, + envelope_from TEXT, + to_addresses_json TEXT NOT NULL DEFAULT '[]', + cc_addresses_json TEXT NOT NULL DEFAULT '[]', + bcc_addresses_json TEXT NOT NULL DEFAULT '[]', + reply_to_addresses_json TEXT NOT NULL DEFAULT '[]', + subject TEXT NOT NULL DEFAULT '', + message_id_header TEXT, + in_reply_to_header TEXT, + references_json TEXT NOT NULL DEFAULT '[]', + headers_json TEXT NOT NULL DEFAULT '{}', + auth_results TEXT, + text_body TEXT, + html_body TEXT, + raw_mime_key TEXT, + raw_size INTEGER NOT NULL DEFAULT 0, + processing_status TEXT NOT NULL, + classification TEXT NOT NULL DEFAULT 'accepted', + classification_reason TEXT, + provider_message_id TEXT, + delivery_status TEXT, + delivery_status_at TEXT, + error TEXT, + received_at TEXT, + sent_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE UNIQUE INDEX idx_email_messages_provider_message_id + ON email_messages(provider_message_id) + WHERE direction = 'outbound' AND provider_message_id IS NOT NULL; + CREATE TABLE email_outbound_provider_index ( + provider TEXT NOT NULL, + provider_message_id TEXT NOT NULL, + user_id TEXT NOT NULL, + message_id TEXT NOT NULL, + inbox_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (provider, provider_message_id), + FOREIGN KEY (message_id) REFERENCES email_messages(id) ON DELETE CASCADE + ); + CREATE UNIQUE INDEX idx_email_outbound_provider_index_message_id + ON email_outbound_provider_index(message_id); + `) + return sqlite +} + +test('deleting email_messages cascades provider index rows', async () => { + const sqlite = createEmailDb() + const db = createD1FromSqlite(sqlite) + const message = await insertEmailMessage({ + db, + message: { + id: 'msg-cascade', + direction: 'outbound', + userId: 'user-cascade', + fromAddress: 'user@example.com', + toAddresses: ['to@example.com'], + subject: 'Cascade', + processingStatus: 'sent', + providerMessageId: 'provider-cascade', + sentAt: '2026-08-01T12:00:00.000Z', + }, + }) + expect( + sqlite + .prepare( + `SELECT COUNT(*) AS count FROM email_outbound_provider_index WHERE message_id = ?`, + ) + .get(message.id) as { count: number }, + ).toEqual({ count: 1 }) + + await db + .prepare(`DELETE FROM email_messages WHERE id = ?`) + .bind(message.id) + .run() + expect( + sqlite + .prepare( + `SELECT COUNT(*) AS count FROM email_outbound_provider_index WHERE message_id = ?`, + ) + .get(message.id) as { count: number }, + ).toEqual({ count: 0 }) +}) + +function wrapDbFailingIndexStatements(sqlite: DatabaseSync): D1Database { + const base = createD1FromSqlite(sqlite) + return { + ...base, + prepare(query: string) { + const prepared = base.prepare(query) + if (!/email_outbound_provider_index/iu.test(query)) { + return prepared + } + return { + ...prepared, + bind(...params: Array) { + const bound = prepared.bind(...params) + return { + ...bound, + async run() { + throw new Error('injected provider-index statement failure') + }, + } + }, + } + }, + async batch(statements: Array<{ run: () => Promise }>) { + return base.batch(statements) + }, + } as unknown as D1Database +} + +test('insert with provider id rolls back message when index statement fails', async () => { + const sqlite = createEmailDb() + const db = wrapDbFailingIndexStatements(sqlite) + const userId = 'user-atomic-insert' + const providerMessageId = 'provider-atomic-insert' + + await expect( + insertEmailMessage({ + db, + message: { + id: 'msg-atomic-insert', + direction: 'outbound', + userId, + fromAddress: 'user@example.com', + toAddresses: ['to@example.com'], + subject: 'Atomic insert', + processingStatus: 'sent', + providerMessageId, + sentAt: '2026-08-01T12:00:00.000Z', + }, + }), + ).rejects.toThrow('injected provider-index statement failure') + + expect( + sqlite.prepare(`SELECT COUNT(*) AS count FROM email_messages`).get() as { + count: number + }, + ).toEqual({ count: 0 }) + expect( + sqlite + .prepare(`SELECT COUNT(*) AS count FROM email_outbound_provider_index`) + .get() as { count: number }, + ).toEqual({ count: 0 }) +}) + +test('updateEmailMessageDelivery rolls back message mutation when index fails', async () => { + const sqlite = createEmailDb() + const healthy = createD1FromSqlite(sqlite) + const message = await insertEmailMessage({ + db: healthy, + message: { + id: 'msg-atomic-update', + direction: 'outbound', + userId: 'user-atomic-update', + fromAddress: 'user@example.com', + toAddresses: ['to@example.com'], + subject: 'Atomic update', + processingStatus: 'stored', + providerMessageId: null, + }, + }) + expect(message.processingStatus).toBe('stored') + expect( + sqlite + .prepare(`SELECT COUNT(*) AS count FROM email_outbound_provider_index`) + .get() as { count: number }, + ).toEqual({ count: 0 }) + + const failingDb = wrapDbFailingIndexStatements(sqlite) + await expect( + updateEmailMessageDelivery({ + db: failingDb, + messageId: message.id, + status: 'sent', + providerMessageId: 'provider-atomic-update', + error: null, + sentAt: '2026-08-01T12:05:00.000Z', + }), + ).rejects.toThrow('injected provider-index statement failure') + + const stored = await getEmailMessageById({ + db: healthy, + userId: 'user-atomic-update', + messageId: message.id, + }) + expect(stored).toMatchObject({ + processingStatus: 'stored', + providerMessageId: null, + sentAt: null, + }) + expect( + sqlite + .prepare(`SELECT COUNT(*) AS count FROM email_outbound_provider_index`) + .get() as { count: number }, + ).toEqual({ count: 0 }) +}) + +test('provider-accepted update cannot leave sent message without index or mark failed from index-only error', async () => { + const sqlite = createEmailDb() + const db = createD1FromSqlite(sqlite) + const userId = 'user-accepted' + const message = await insertEmailMessage({ + db, + message: { + id: 'msg-accepted', + direction: 'outbound', + userId, + fromAddress: 'user@example.com', + toAddresses: ['to@example.com'], + subject: 'Accepted', + processingStatus: 'stored', + }, + }) + + const providerMessageId = 'provider-accepted' + const batchSpy = vi.spyOn(db, 'batch') + await updateEmailMessageDelivery({ + db, + messageId: message.id, + status: 'sent', + providerMessageId, + error: null, + sentAt: '2026-08-01T12:10:00.000Z', + }) + expect(batchSpy).toHaveBeenCalledTimes(1) + const batchStatements = batchSpy.mock.calls[0]?.[0] ?? [] + expect(batchStatements.length).toBeGreaterThan(1) + + const stored = await getEmailMessageById({ + db, + userId, + messageId: message.id, + }) + expect(stored).toMatchObject({ + processingStatus: 'sent', + providerMessageId, + error: null, + }) + expect( + await getOutboundEmailMessageByProviderMessageId({ + db, + providerMessageId, + }), + ).toMatchObject({ id: message.id, userId }) + expect( + ( + sqlite + .prepare( + `SELECT provider, user_id, message_id + FROM email_outbound_provider_index + WHERE provider_message_id = ?`, + ) + .get(providerMessageId) as { + provider: string + user_id: string + message_id: string + } | null + )?.provider, + ).toBe(emailOutboundProviderCloudflare) + + // A post-commit index-only failure cannot exist: sync statements were in the + // same accepted batch as the authoritative status update. + expect(stored?.processingStatus).not.toBe('failed') +}) diff --git a/packages/worker/src/email/outbound-provider-index-migration.node.test.ts b/packages/worker/src/email/outbound-provider-index-migration.node.test.ts index 8bdf99e29c..22c51e3bf3 100644 --- a/packages/worker/src/email/outbound-provider-index-migration.node.test.ts +++ b/packages/worker/src/email/outbound-provider-index-migration.node.test.ts @@ -10,6 +10,7 @@ const migrationUrl = new URL( function createPreMigrationDb() { const db = new DatabaseSync(':memory:') + db.exec('PRAGMA foreign_keys = ON') db.exec(` CREATE TABLE email_messages ( id TEXT PRIMARY KEY, @@ -84,7 +85,7 @@ test('provider index migration backfills outbound provider ids including system '2026-08-01T00:04:00.000Z', '2026-08-01T00:04:00.000Z', ) - }).toThrow(/UNIQUE constraint failed/) + }).toThrow(/UNIQUE constraint failed|FOREIGN KEY/u) expect(() => { db.prepare( @@ -147,3 +148,41 @@ test('provider index migration backfill counts match linked outbound messages', expect(indexed.count).toBe(2) expect(missingFromIndex.count).toBe(0) }) + +test('provider index message_id FK cascades on email_messages delete', () => { + const db = createPreMigrationDb() + db.exec(` + INSERT INTO email_messages ( + id, direction, user_id, inbox_id, provider_message_id, created_at, updated_at + ) VALUES + ('keep', 'outbound', 'user-1', NULL, 'prov-keep', + '2026-08-01T00:00:00.000Z', '2026-08-01T00:00:00.000Z'), + ('drop', 'outbound', 'user-1', NULL, 'prov-drop', + '2026-08-01T00:01:00.000Z', '2026-08-01T00:01:00.000Z'); + `) + db.exec(readFileSync(migrationUrl, 'utf8')) + + const fk = db + .prepare(`PRAGMA foreign_key_list(email_outbound_provider_index)`) + .all() as Array<{ table: string; from: string; on_delete: string }> + expect(fk).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + table: 'email_messages', + from: 'message_id', + on_delete: 'CASCADE', + }), + ]), + ) + + db.prepare(`DELETE FROM email_messages WHERE id = 'drop'`).run() + expect( + db + .prepare( + `SELECT provider_message_id + FROM email_outbound_provider_index + ORDER BY provider_message_id`, + ) + .all(), + ).toEqual([{ provider_message_id: 'prov-keep' }]) +}) diff --git a/packages/worker/src/email/outbound-provider-index.workers.test.ts b/packages/worker/src/email/outbound-provider-index.workers.test.ts index 20339f18a3..52f9ea480b 100644 --- a/packages/worker/src/email/outbound-provider-index.workers.test.ts +++ b/packages/worker/src/email/outbound-provider-index.workers.test.ts @@ -9,11 +9,13 @@ import { getOutboundProviderIndexRow, loadOutboundProviderIndexParityReport, } from './outbound-provider-index.ts' -import { sendOutboundEmail } from './outbound.ts' +import { OutboundEmailPersistenceError, sendOutboundEmail } from './outbound.ts' import { deleteEmailMessageById, + getEmailMessageById, getOutboundEmailMessageByProviderMessageId, insertEmailMessage, + listEmailDeliveryEvents, updateEmailMessageDelivery, } from './repo.ts' import { ensureEmailTestSchema } from './test-schema.ts' @@ -314,6 +316,21 @@ test('parity report flags missing and mismatched index rows', async () => { ) .bind(message.id) .run() + // Index row points at a real message that is not a linked outbound provider + // row (no provider_message_id), so FK stays satisfied while parity flags + // missing_from_messages. + const orphanHost = await insertEmailMessage({ + db: env.APP_DB, + message: { + direction: 'outbound', + userId, + fromAddress: 'user@inbox.example.com', + toAddresses: ['recipient@example.net'], + subject: 'Orphan host', + processingStatus: 'stored', + providerMessageId: null, + }, + }) await env.APP_DB.prepare( `INSERT INTO email_outbound_provider_index ( provider, provider_message_id, user_id, message_id, inbox_id, @@ -324,7 +341,7 @@ test('parity report flags missing and mismatched index rows', async () => { emailOutboundProviderCloudflare, `orphan-${crypto.randomUUID()}`, userId, - `missing-message-${crypto.randomUUID()}`, + orphanHost.id, '2026-08-01T15:00:00.000Z', '2026-08-01T15:00:00.000Z', ) @@ -377,3 +394,87 @@ test('sendOutboundEmail dual-writes the provider index on terminal success', asy }), ).toMatchObject({ parity: true, linkedMessageCount: 1, indexCount: 1 }) }, 30_000) + +test('sendOutboundEmail retains provider acceptance when terminal D1 persistence fails', async () => { + silenceIncidentalRuntimeWarnings([ + 'email-outbound-terminal-persistence-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + const email = `idx-persist-fail-${crypto.randomUUID()}@example.com` + const userId = await seedVerifiedAccount(email) + const acceptedProviderMessageId = `provider-accepted-${crypto.randomUUID()}` + let providerCalls = 0 + const failingDb = new Proxy(env.APP_DB, { + get(target, property, receiver) { + if (property === 'batch') { + return async ( + statements: Parameters[0], + ): Promise> => { + // Fail only terminal update+index batches (2+ statements). + if (statements.length >= 2) { + throw new Error('injected terminal persistence failure') + } + return target.batch(statements) + } + } + const value = Reflect.get(target, property, receiver) + return typeof value === 'function' + ? (value as (...args: Array) => unknown).bind(target) + : value + }, + }) + + const error = await sendOutboundEmail({ + env: { + ...createBindingSendEnv(), + APP_DB: failingDb, + EMAIL: { + async send() { + providerCalls += 1 + return { messageId: acceptedProviderMessageId } + }, + }, + }, + userId, + accountEmail: email, + recipientPolicy: 'self', + subject: 'Persistence failure', + text: 'hello', + }).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(OutboundEmailPersistenceError) + expect(error).toMatchObject({ + messageId: expect.any(String), + providerMessageId: acceptedProviderMessageId, + }) + expect(providerCalls).toBe(1) + + const messageId = (error as OutboundEmailPersistenceError).messageId + expect( + await getEmailMessageById({ + db: env.APP_DB, + userId, + messageId, + }), + ).toMatchObject({ + processingStatus: 'stored', + providerMessageId: null, + error: null, + }) + expect( + await getOutboundProviderIndexRow({ + db: env.APP_DB, + providerMessageId: acceptedProviderMessageId, + }), + ).toBeNull() + const events = await listEmailDeliveryEvents({ + db: env.APP_DB, + userId, + messageId, + limit: 20, + }) + expect(events.map((event) => event.eventType).sort()).toEqual([ + 'send_requested', + ]) + expect(events.some((event) => event.eventType === 'failed')).toBe(false) +}, 30_000) diff --git a/packages/worker/src/email/outbound.ts b/packages/worker/src/email/outbound.ts index c03d1ca88c..d3481bc123 100644 --- a/packages/worker/src/email/outbound.ts +++ b/packages/worker/src/email/outbound.ts @@ -8,6 +8,7 @@ import { sendCloudflareEmail } from '#app/email/cloudflare-email.ts' import { isAccountEmailVerified } from '#worker/identity/email-verification-state.ts' import { normalizeEmail } from '#worker/identity/normalize-email.ts' import { withAccountWriteLease } from '#worker/account/deletion-state.ts' +import { runD1WithRetry } from '#worker/d1-retry.ts' import { McpCallerError } from '#mcp/caller-error.ts' import { assertWithinEntitlement, @@ -130,6 +131,29 @@ export type EmailSendResult = { error: string | null } +/** + * Provider accepted the send, but D1 terminal persistence (message + provider + * index, and/or the `sent` delivery event) failed after bounded retries. The + * message must not be marked `failed` and must not be resent — repair D1. + */ +export class OutboundEmailPersistenceError extends Error { + override name = 'OutboundEmailPersistenceError' + readonly messageId: string + readonly providerMessageId: string | null + constructor(input: { + messageId: string + providerMessageId: string | null + cause?: unknown + }) { + super( + `Provider accepted outbound email ${input.messageId} (providerMessageId=${input.providerMessageId ?? 'null'}) but D1 terminal persistence failed after retries. Do not resend; repair D1/index for this message.`, + { cause: input.cause }, + ) + this.messageId = input.messageId + this.providerMessageId = input.providerMessageId + } +} + function resolveSelfRecipients(input: { to: string | Array | null | undefined accountEmail: string @@ -716,54 +740,119 @@ export async function sendOutboundEmail( const sendStartedAtMs = Date.now() let sendOutcome: 'success' | 'error' = 'success' try { - const bindingResult = await sendViaBinding({ - env: input.env, - from, - to, - subject, - text, - html, - replyTo: input.replyTo - ? (normalizeEmailAddress(input.replyTo) ?? undefined) - : undefined, - headers: providerHeaders, - attachments, - }) - const providerMessageId = bindingResult.sent - ? bindingResult.messageId - : await sendViaRestFallback({ - env: input.env, - from, - to, - subject, - text, - html, - replyTo: input.replyTo - ? (normalizeEmailAddress(input.replyTo) ?? undefined) - : undefined, - headers: providerHeaders, - attachments, - }) - await updateEmailMessageDelivery({ - db: input.env.APP_DB, - messageId: message.id, - status: 'sent', - providerMessageId, - error: null, - sentAt: new Date().toISOString(), - userId: input.userId, - inboxId: message.inboxId, - }) - await insertEmailDeliveryEvent({ - db: input.env.APP_DB, - messageId: message.id, - userId: input.userId, - inboxId: null, - eventType: 'sent', - provider: 'cloudflare-email', - providerMessageId, - detail: { providerMessageId }, - }) + let acceptedProviderMessageId: string | null + try { + const bindingResult = await sendViaBinding({ + env: input.env, + from, + to, + subject, + text, + html, + replyTo: input.replyTo + ? (normalizeEmailAddress(input.replyTo) ?? undefined) + : undefined, + headers: providerHeaders, + attachments, + }) + acceptedProviderMessageId = bindingResult.sent + ? bindingResult.messageId + : await sendViaRestFallback({ + env: input.env, + from, + to, + subject, + text, + html, + replyTo: input.replyTo + ? (normalizeEmailAddress(input.replyTo) ?? undefined) + : undefined, + headers: providerHeaders, + attachments, + }) + } catch (error) { + // Provider did not accept the send. Safe to mark failed / clear id. + sendOutcome = 'error' + const messageText = getErrorMessage(error) + await updateEmailMessageDelivery({ + db: input.env.APP_DB, + messageId: message.id, + status: 'failed', + providerMessageId: null, + error: messageText, + sentAt: null, + }).catch((updateError) => { + console.warn( + 'email-delivery-failure-status-update-failed', + updateError, + ) + }) + await insertEmailDeliveryEvent({ + db: input.env.APP_DB, + messageId: message.id, + userId: input.userId, + inboxId: null, + eventType: 'failed', + provider: 'cloudflare-email', + providerMessageId: null, + detail: { error: messageText }, + }).catch((eventError) => { + console.warn('email-delivery-failure-event-insert-failed', eventError) + }) + await mirrorMailboxGraph() + return { + message: + (await getEmailMessageById({ + db: input.env.APP_DB, + userId: input.userId, + messageId: message.id, + })) ?? message, + providerMessageId: null, + status: 'failed', + error: messageText, + } + } + + // Provider accepted — never enter the send-failure branch, clear the + // provider id, or call the provider again. Persist terminal D1/index + // state with bounded D1 retries; exhausted retries are operational. + const sentAt = new Date().toISOString() + try { + await runD1WithRetry(() => + updateEmailMessageDelivery({ + db: input.env.APP_DB, + messageId: message.id, + status: 'sent', + providerMessageId: acceptedProviderMessageId, + error: null, + sentAt, + }), + ) + await runD1WithRetry(() => + insertEmailDeliveryEvent({ + db: input.env.APP_DB, + messageId: message.id, + userId: input.userId, + inboxId: null, + eventType: 'sent', + provider: 'cloudflare-email', + providerMessageId: acceptedProviderMessageId, + detail: { providerMessageId: acceptedProviderMessageId }, + }), + ) + } catch (error) { + console.warn('email-outbound-terminal-persistence-failed', { + messageId: message.id, + providerMessageId: acceptedProviderMessageId, + error, + }) + throw new OutboundEmailPersistenceError({ + messageId: message.id, + providerMessageId: acceptedProviderMessageId, + cause: error, + }) + } + await mirrorMailboxGraph() return { message: await requireStoredEmailMessage({ @@ -771,47 +860,10 @@ export async function sendOutboundEmail( userId: input.userId, messageId: message.id, }), - providerMessageId, + providerMessageId: acceptedProviderMessageId, status: 'sent', error: null, } - } catch (error) { - sendOutcome = 'error' - const messageText = getErrorMessage(error) - await updateEmailMessageDelivery({ - db: input.env.APP_DB, - messageId: message.id, - status: 'failed', - providerMessageId: null, - error: messageText, - sentAt: null, - }).catch((updateError) => { - console.warn('email-delivery-failure-status-update-failed', updateError) - }) - await insertEmailDeliveryEvent({ - db: input.env.APP_DB, - messageId: message.id, - userId: input.userId, - inboxId: null, - eventType: 'failed', - provider: 'cloudflare-email', - providerMessageId: null, - detail: { error: messageText }, - }).catch((eventError) => { - console.warn('email-delivery-failure-event-insert-failed', eventError) - }) - await mirrorMailboxGraph() - return { - message: - (await getEmailMessageById({ - db: input.env.APP_DB, - userId: input.userId, - messageId: message.id, - })) ?? message, - providerMessageId: null, - status: 'failed', - error: messageText, - } } finally { if (input.userId) { await recordUsage(input.env, { diff --git a/packages/worker/src/email/repo.ts b/packages/worker/src/email/repo.ts index 9e3847dcf3..2bb2126167 100644 --- a/packages/worker/src/email/repo.ts +++ b/packages/worker/src/email/repo.ts @@ -1274,14 +1274,11 @@ export async function deleteEmailMessageById(input: { ] // Atomic batch: a partial delete (attachments gone, message left) would // lose the storage_key values needed to delete the R2 blobs on retry. - // The derived provider reverse index is cleared with the message row. + // Provider-index rows cascade via FK ON DELETE CASCADE from email_messages. await input.db.batch([ input.db .prepare(`DELETE FROM email_attachments WHERE message_id = ?`) .bind(input.messageId), - input.db - .prepare(`DELETE FROM email_outbound_provider_index WHERE message_id = ?`) - .bind(input.messageId), input.db .prepare(`DELETE FROM email_messages WHERE id = ?`) .bind(input.messageId), diff --git a/packages/worker/src/email/system-email.ts b/packages/worker/src/email/system-email.ts index 6ada88bd4e..a1aa4388a0 100644 --- a/packages/worker/src/email/system-email.ts +++ b/packages/worker/src/email/system-email.ts @@ -1,6 +1,5 @@ import { utcDayKey } from '@kody-internal/shared/date-keys.ts' import { systemEmailOwnerId } from './email-owner.ts' -import { deleteOutboundProviderIndexByMessageIds } from './outbound-provider-index.ts' import { maxRawMimeBytes } from './parser.ts' import { buildPlatformEmailAddress } from './platform-address.ts' import { @@ -324,10 +323,7 @@ async function deleteSystemEmailMessagesByIds(input: { ) .bind(...deletableIds) .run() - await deleteOutboundProviderIndexByMessageIds({ - db: input.db, - messageIds: deletableIds, - }) + // Provider-index rows cascade via FK ON DELETE CASCADE from email_messages. await input.db .prepare( `DELETE FROM email_messages diff --git a/packages/worker/src/email/test-schema.ts b/packages/worker/src/email/test-schema.ts index 1b248447e3..6d35927cf4 100644 --- a/packages/worker/src/email/test-schema.ts +++ b/packages/worker/src/email/test-schema.ts @@ -178,7 +178,8 @@ WHERE direction = 'outbound' inbox_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - PRIMARY KEY (provider, provider_message_id) + PRIMARY KEY (provider, provider_message_id), + FOREIGN KEY (message_id) REFERENCES email_messages(id) ON DELETE CASCADE );`, `CREATE INDEX IF NOT EXISTS idx_email_outbound_provider_index_user_id ON email_outbound_provider_index(user_id);`, diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 12d28901c3..9cf767641c 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -526,7 +526,7 @@ }, { "filename": "0128-email-outbound-provider-index.sql", - "sha256": "84301bf6d3c533db15f94aaabde2a34d7ecb1fb196578666b6bfd3c2dffd3408" + "sha256": "b51c4f9cc4416cd81c1c1648311e0267e28a208d386d4729295b1d3dad54ed5a" } ] } From f31c296b3508e9bb8af88f36f10528d2601e3a7e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 13:46:27 +0000 Subject: [PATCH 46/62] feat(email): add Mailbox inbound ledger CAS RPCs Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 23 +- packages/worker/src/email/mailbox-do.ts | 316 ++++++- .../email/mailbox-inbound-effect-ledger.ts | 605 +++++++++++++ .../email/mailbox-inbound-ledger-shared.ts | 375 ++++++++ .../email/mailbox-inbound-ledger.node.test.ts | 61 ++ .../src/email/mailbox-inbound-ledger.ts | 807 ++++++++++++++++++ .../mailbox-inbound-ledger.workers.test.ts | 619 ++++++++++++++ packages/worker/src/email/mailbox-schema.ts | 43 +- packages/worker/src/email/mailbox-store.ts | 5 + packages/worker/src/email/mailbox-types.ts | 25 +- 10 files changed, 2871 insertions(+), 8 deletions(-) create mode 100644 packages/worker/src/email/mailbox-inbound-effect-ledger.ts create mode 100644 packages/worker/src/email/mailbox-inbound-ledger-shared.ts create mode 100644 packages/worker/src/email/mailbox-inbound-ledger.node.test.ts create mode 100644 packages/worker/src/email/mailbox-inbound-ledger.ts create mode 100644 packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index bdf83469ea..4ba19272e9 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -697,8 +697,8 @@ persists `ownerId` on first write and rejects cross-owner RPCs. That persisted owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` / `emailAttachmentBlobKey`). -**SQLite ownership** (schema version in `mailbox_meta`; ships as -`mailboxSchemaVersion = 1`): +**SQLite ownership** (schema version in `mailbox_meta`; current +`mailboxSchemaVersion = 2`): - `mailbox_owner_identity` — singleton `owner_id` for blob-key validation and cross-user write rejection @@ -707,6 +707,25 @@ owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` tables they will eventually replace) - latest per-message delivery status on `email_messages.delivery_status`, kept separate from send-request `processing_status` +- **Schema v2 (warm-safe):** additive inbound-ledger due-work indexes on + `email_delivery_events` (reconcile/retry/stale-state/dedupe-provider). Cold + objects install the full DDL; warm v1 objects run `CREATE INDEX IF NOT EXISTS` + only. No destructive ALTERs. + +**Additive inbound ledger CAS (step 2a — no authority flip):** owner-bound +atomic RPCs in `mailbox-inbound-ledger.ts` / `mailbox-inbound-effect-ledger.ts` +mirror D1 USER transitions from `inbound-delivery.ts` / `inbound-effects.ts` +(get delivery + active fingerprint window; claim/rewrite dedupe; insert charged +pending with explicit `inserted`/`existed`; claim/release storage lease; mark +rejected/received with lease/finalization CAS; prune dedupe; defer reconcile; +claim/complete usage effect; claim/complete/fail subscription effect with +retry/dead-letter/ suppression; list due stale/effect work). Mutations keep +promoted columns and `detail_json` in sync and set canonical `updated_at`. The +DO does **not** perform external usage recording or subscription dispatch. +Mirror `upsertDeliveryEvent` / `upsertDeliveryEvents` remain the compatibility +write API. **These CAS RPCs are not live-wired** — D1 stays sole write authority +for inbound ledger/effects. `system:email` stays on D1 and is never written into +per-user Mailbox objects. **Mirror write contract:** `mirrorMessage`, `upsertDeliveryEvent`, and `upsertDeliveryEvents` take complete snapshots — every persisted field is diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index e06e65f57c..f26aa4f1a7 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -21,6 +21,29 @@ import { touchMailboxThread, updateMailboxMessageDelivery, } from './mailbox-mutations.ts' +import { + claimMailboxInboundDeliveryStorage, + claimMailboxInboundDeliveryWindow, + deferMailboxInboundDeliveryReconciliation, + getMailboxInboundDelivery, + getMailboxInboundDeliveryWindow, + insertMailboxChargedPendingInboundDelivery, + listMailboxDueStaleInboundDeliveries, + markMailboxInboundDeliveryReceived, + markMailboxInboundDeliveryRejected, + pruneMailboxExpiredInboundDedupePointers, + releaseMailboxInboundDeliveryStorage, + type MailboxInboundDeliveryInsertInput, + type MailboxInboundDeliverySnapshot, +} from './mailbox-inbound-ledger.ts' +import { + claimMailboxInboundSubscriptionEffect, + claimMailboxInboundUsageEffect, + completeMailboxInboundSubscriptionEffect, + completeMailboxInboundUsageEffect, + failMailboxInboundSubscriptionEffect, + listMailboxDueInboundEffectWork, +} from './mailbox-inbound-effect-ledger.ts' import { assertMailboxNonEmptyString, mailboxUpsertDeliveryEventsMax, @@ -36,6 +59,7 @@ import { type MailboxDeliveryEventInput, type MailboxDeliveryEventRecord, type MailboxExportResult, + type MailboxInboundDeliveryState, type MailboxListMessagesInput, type MailboxMessageInput, type MailboxMessageRecord, @@ -60,7 +84,9 @@ import { * external attachment bytes stay in `EMAIL_BLOBS`; rows retain keys. * `system:email` stays in D1 by design. * - * Scaffold only: dual-write / cutover callers are not wired here. + * Dual-write / read-cutover live paths are wired elsewhere. Additive step 2a + * inbound ledger CAS RPCs are exposed but not live-wired (D1 remains + * authority for inbound ledger/effects). */ class MailboxBase extends DurableObject implements MailboxRpc { @@ -481,6 +507,294 @@ class MailboxBase extends DurableObject implements MailboxRpc { return this.store.listBlobReferences(input) } + /** + * Additive step 2a inbound ledger CAS RPCs. Not live-wired — D1 remains + * authority. Mirror `upsertDeliveryEvent(s)` stay the compatibility path. + */ + + async getInboundDelivery(input: { + ownerId: string + deliveryId: string + }): Promise { + this.store.assertOwner(input.ownerId) + return getMailboxInboundDelivery(this.ctx.storage.sql, input.deliveryId) + } + + async getInboundDeliveryWindow(input: { + ownerId: string + fingerprint: string + now?: string + }): Promise { + this.store.assertOwner(input.ownerId) + return getMailboxInboundDeliveryWindow(this.ctx.storage.sql, input) + } + + async claimInboundDeliveryWindow(input: { + ownerId: string + delivery: MailboxInboundDeliveryInsertInput + now?: string + }): Promise { + let result!: MailboxInboundDeliverySnapshot + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = claimMailboxInboundDeliveryWindow(this.ctx.storage.sql, input) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return result + } + + async insertChargedPendingInboundDelivery(input: { + ownerId: string + delivery: MailboxInboundDeliveryInsertInput + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = insertMailboxChargedPendingInboundDelivery( + this.ctx.storage.sql, + input, + ) + }) + this.markRetentionDirty() + await this.ensureRetentionAlarm() + return result + } + + async claimInboundDeliveryStorage(input: { + ownerId: string + deliveryId: string + expectedAttachmentCount: number + usageStartedAt?: string | null + now?: string + }) { + let result!: Awaited> + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = claimMailboxInboundDeliveryStorage(this.ctx.storage.sql, input) + }) + if (result.status === 'claimed') { + this.markRetentionDirty() + await this.ensureRetentionAlarm() + } + return result + } + + async releaseInboundDeliveryStorage(input: { + ownerId: string + deliveryId: string + storageLease: string + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = releaseMailboxInboundDeliveryStorage(this.ctx.storage.sql, input) + }) + if (result.status === 'released') { + this.markRetentionDirty() + await this.ensureRetentionAlarm() + } + return result + } + + async markInboundDeliveryRejected(input: { + ownerId: string + deliveryId: string + reason: string + expectedStorageLease?: string | null + expectedState?: MailboxInboundDeliveryState + now?: string + }) { + let result!: Awaited> + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = markMailboxInboundDeliveryRejected(this.ctx.storage.sql, input) + }) + if (result.status === 'rejected') { + this.markRetentionDirty() + await this.ensureRetentionAlarm() + } + return result + } + + async markInboundDeliveryReceived(input: { + ownerId: string + deliveryId: string + storageLease: string + usageDurationMs: number + usageMonth: string + usageBytes: number + now?: string + }) { + let result!: Awaited> + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = markMailboxInboundDeliveryReceived(this.ctx.storage.sql, input) + }) + if (result.status === 'received') { + this.markRetentionDirty() + await this.ensureRetentionAlarm() + } + return result + } + + async pruneExpiredInboundDedupePointers(input: { + ownerId: string + now?: string + limit?: number + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = pruneMailboxExpiredInboundDedupePointers( + this.ctx.storage.sql, + input, + ) + }) + if (result.pruned > 0) { + this.markRetentionDirty() + await this.ensureRetentionAlarm() + } + return result + } + + async deferInboundDeliveryReconciliation(input: { + ownerId: string + deliveryId: string + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = deferMailboxInboundDeliveryReconciliation( + this.ctx.storage.sql, + input, + ) + }) + if (result.status === 'deferred') { + this.markRetentionDirty() + await this.ensureRetentionAlarm() + } + return result + } + + async claimInboundUsageEffect(input: { + ownerId: string + deliveryId: string + expectedFinalizationToken?: string | null + now?: string + }) { + let result!: Awaited> + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = claimMailboxInboundUsageEffect(this.ctx.storage.sql, input) + }) + return result + } + + async completeInboundUsageEffect(input: { + ownerId: string + deliveryId: string + usageEffectLease: string + mode: 'recorded' | 'suppressed' + usageMonth: string + usageBytes: number + usageDurationMs: number + now?: string + }) { + let result!: Awaited> + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = completeMailboxInboundUsageEffect(this.ctx.storage.sql, input) + }) + return result + } + + async claimInboundSubscriptionEffect(input: { + ownerId: string + deliveryId: string + expectedFinalizationToken?: string | null + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = claimMailboxInboundSubscriptionEffect( + this.ctx.storage.sql, + input, + ) + }) + return result + } + + async completeInboundSubscriptionEffect(input: { + ownerId: string + deliveryId: string + subscriptionEffectLease: string + mode: 'complete' | 'suppressed' + suppressionReason?: string | null + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = completeMailboxInboundSubscriptionEffect( + this.ctx.storage.sql, + input, + ) + }) + return result + } + + async failInboundSubscriptionEffect(input: { + ownerId: string + deliveryId: string + subscriptionEffectLease: string + error: string + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = failMailboxInboundSubscriptionEffect(this.ctx.storage.sql, input) + }) + return result + } + + async listDueStaleInboundDeliveries(input: { + ownerId: string + now?: string + limit?: number + }) { + this.store.assertOwner(input.ownerId) + return listMailboxDueStaleInboundDeliveries(this.ctx.storage.sql, input) + } + + async listDueInboundEffectWork(input: { + ownerId: string + now?: string + limit?: number + }) { + this.store.assertOwner(input.ownerId) + return listMailboxDueInboundEffectWork(this.ctx.storage.sql, input) + } + /** * Clear SQLite state only. During expand, D1 deletion remains authoritative * for R2 objects. diff --git a/packages/worker/src/email/mailbox-inbound-effect-ledger.ts b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts new file mode 100644 index 0000000000..d205f8671e --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts @@ -0,0 +1,605 @@ +/** + * Additive Mailbox USER inbound effect CAS primitives (step 2a). + * + * Lease claim / complete / fail for usage and subscription effects. Does not + * perform external usage recording or subscription dispatch — callers do that + * after a successful claim. Part of the inbound ledger surface; see + * `mailbox-inbound-ledger.ts`. + */ + +import { assertMailboxNonEmptyString } from './mailbox-types.ts' +import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' +import { + detailJsonFromMailboxInboundSnapshot, + mailboxEffectRetryMs, + mailboxInboundProvider, + mailboxSubscriptionEffectLeaseMs, + mailboxUsageEffectLeaseMs, + needsMailboxEffectReconcile, + normalizeMailboxInboundLimit, + normalizeMailboxInboundNow, + readMailboxInboundDeliveryById, + resolveMailboxSubscriptionEffectFailure, + snapshotFromMailboxDeliveryEventRow, + type MailboxInboundDeliverySnapshot, +} from './mailbox-inbound-ledger.ts' + +export type MailboxClaimInboundUsageEffectResult = + | { status: 'claimed'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'already-complete'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'not-claimable'; delivery: MailboxInboundDeliverySnapshot | null } + +export type MailboxCompleteInboundUsageEffectResult = + | { status: 'recorded'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'suppressed'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'already-complete'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'lease-lost' } + +export type MailboxClaimInboundSubscriptionEffectResult = + | { status: 'claimed'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'already-complete'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'not-claimable'; delivery: MailboxInboundDeliverySnapshot | null } + +export type MailboxCompleteInboundSubscriptionEffectResult = + | { status: 'complete'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'suppressed'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'already-complete'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'lease-lost' } + +export type MailboxFailInboundSubscriptionEffectResult = + | { status: 'retry'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'dead-letter'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'lease-lost' } + +export type MailboxListDueInboundEffectWorkResult = { + deliveries: Array +} + +export type MailboxInboundEffectLedgerRpc = { + claimInboundUsageEffect: (input: { + ownerId: string + deliveryId: string + expectedFinalizationToken?: string | null + now?: string + }) => Promise + completeInboundUsageEffect: (input: { + ownerId: string + deliveryId: string + usageEffectLease: string + mode: 'recorded' | 'suppressed' + usageMonth: string + usageBytes: number + usageDurationMs: number + now?: string + }) => Promise + claimInboundSubscriptionEffect: (input: { + ownerId: string + deliveryId: string + expectedFinalizationToken?: string | null + now?: string + }) => Promise + completeInboundSubscriptionEffect: (input: { + ownerId: string + deliveryId: string + subscriptionEffectLease: string + mode: 'complete' | 'suppressed' + suppressionReason?: string | null + now?: string + }) => Promise + failInboundSubscriptionEffect: (input: { + ownerId: string + deliveryId: string + subscriptionEffectLease: string + error: string + now?: string + }) => Promise + listDueInboundEffectWork: (input: { + ownerId: string + now?: string + limit?: number + }) => Promise +} + +export function claimMailboxInboundUsageEffect( + sql: SqlStorage, + input: { + deliveryId: string + expectedFinalizationToken?: string | null + now?: string + }, +): MailboxClaimInboundUsageEffectResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current || current.state !== 'received') { + return { status: 'not-claimable', delivery: current } + } + if ( + input.expectedFinalizationToken != null && + current.finalizationToken !== input.expectedFinalizationToken + ) { + return { status: 'not-claimable', delivery: current } + } + if ( + current.usageEffectRecordedAt != null || + current.usageEffectSuppressedAt + ) { + return { status: 'already-complete', delivery: current } + } + if (current.usageEffectRetryAt != null && current.usageEffectRetryAt > now) { + return { status: 'not-claimable', delivery: current } + } + const expiredBefore = new Date( + Date.parse(now) - mailboxUsageEffectLeaseMs, + ).toISOString() + if ( + current.usageEffectLease != null && + current.usageEffectLeaseAt != null && + current.usageEffectLeaseAt >= expiredBefore + ) { + return { status: 'not-claimable', delivery: current } + } + const usageEffectLease = crypto.randomUUID() + const next: MailboxInboundDeliverySnapshot = { + ...current, + usageEffectLease, + usageEffectLeaseAt: now, + updatedAt: now, + } + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + usage_effect_lease = ?, + usage_effect_lease_at = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND event_type = 'received' + AND state = 'received' + AND usage_effect_recorded_at IS NULL + AND usage_effect_suppressed_at IS NULL + AND (usage_effect_retry_at IS NULL OR usage_effect_retry_at <= ?) + AND (usage_effect_lease IS NULL OR usage_effect_lease_at < ?) + AND (? IS NULL OR finalization_token = ?)`, + detailJsonFromMailboxInboundSnapshot(next), + usageEffectLease, + now, + now, + deliveryId, + mailboxInboundProvider, + now, + expiredBefore, + input.expectedFinalizationToken ?? null, + input.expectedFinalizationToken ?? null, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (cursor.rowsWritten > 0 && after?.usageEffectLease === usageEffectLease) { + return { status: 'claimed', delivery: after } + } + if (after?.usageEffectRecordedAt || after?.usageEffectSuppressedAt) { + return { status: 'already-complete', delivery: after } + } + return { status: 'not-claimable', delivery: after } +} + +export function completeMailboxInboundUsageEffect( + sql: SqlStorage, + input: { + deliveryId: string + usageEffectLease: string + mode: 'recorded' | 'suppressed' + usageMonth: string + usageBytes: number + usageDurationMs: number + now?: string + }, +): MailboxCompleteInboundUsageEffectResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const usageEffectLease = assertMailboxNonEmptyString( + input.usageEffectLease, + 'usageEffectLease', + ) + const usageMonth = assertMailboxNonEmptyString(input.usageMonth, 'usageMonth') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current) return { status: 'lease-lost' } + if (current.usageEffectRecordedAt || current.usageEffectSuppressedAt) { + return { status: 'already-complete', delivery: current } + } + if (current.usageEffectLease !== usageEffectLease) { + return { status: 'lease-lost' } + } + const usageBytes = Math.max(0, Math.round(input.usageBytes)) + const usageDurationMs = Math.max(0, Math.round(input.usageDurationMs)) + const next: MailboxInboundDeliverySnapshot = { + ...current, + usageMonth, + usageBytes, + usageDurationMs, + updatedAt: now, + ...(input.mode === 'recorded' + ? { usageEffectRecordedAt: now } + : { usageEffectSuppressedAt: now }), + } + delete next.usageEffectLease + delete next.usageEffectLeaseAt + delete next.usageEffectRetryAt + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + usage_effect_recorded_at = ?, + usage_effect_suppressed_at = ?, + usage_month = ?, + usage_bytes = ?, + usage_duration_ms = ?, + usage_effect_lease = NULL, + usage_effect_lease_at = NULL, + usage_effect_retry_at = NULL, + needs_effect_reconcile = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND usage_effect_lease = ? + AND usage_effect_recorded_at IS NULL + AND usage_effect_suppressed_at IS NULL`, + detailJsonFromMailboxInboundSnapshot(next), + input.mode === 'recorded' ? now : null, + input.mode === 'suppressed' ? now : null, + usageMonth, + usageBytes, + usageDurationMs, + needsMailboxEffectReconcile(next) ? 1 : 0, + now, + deliveryId, + mailboxInboundProvider, + usageEffectLease, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (!after) return { status: 'lease-lost' } + if ( + cursor.rowsWritten > 0 && + input.mode === 'recorded' && + after.usageEffectRecordedAt + ) { + return { status: 'recorded', delivery: after } + } + if ( + cursor.rowsWritten > 0 && + input.mode === 'suppressed' && + after.usageEffectSuppressedAt + ) { + return { status: 'suppressed', delivery: after } + } + if (after.usageEffectRecordedAt || after.usageEffectSuppressedAt) { + return { status: 'already-complete', delivery: after } + } + return { status: 'lease-lost' } +} + +export function claimMailboxInboundSubscriptionEffect( + sql: SqlStorage, + input: { + deliveryId: string + expectedFinalizationToken?: string | null + now?: string + }, +): MailboxClaimInboundSubscriptionEffectResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current || current.state !== 'received') { + return { status: 'not-claimable', delivery: current } + } + if ( + input.expectedFinalizationToken != null && + current.finalizationToken !== input.expectedFinalizationToken + ) { + return { status: 'not-claimable', delivery: current } + } + if ( + current.subscriptionEffectState === 'complete' || + current.subscriptionEffectState === 'dead-letter' + ) { + return { status: 'already-complete', delivery: current } + } + if ( + current.subscriptionEffectRetryAt != null && + current.subscriptionEffectRetryAt > now + ) { + return { status: 'not-claimable', delivery: current } + } + const expiredBefore = new Date( + Date.parse(now) - mailboxSubscriptionEffectLeaseMs, + ).toISOString() + if ( + current.subscriptionEffectState === 'processing' && + current.subscriptionEffectLeaseAt != null && + current.subscriptionEffectLeaseAt >= expiredBefore + ) { + return { status: 'not-claimable', delivery: current } + } + const subscriptionEffectLease = crypto.randomUUID() + const next: MailboxInboundDeliverySnapshot = { + ...current, + subscriptionEffectState: 'processing', + subscriptionEffectLease, + subscriptionEffectLeaseAt: now, + updatedAt: now, + } + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + subscription_effect_state = 'processing', + subscription_effect_lease = ?, + subscription_effect_lease_at = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND event_type = 'received' + AND state = 'received' + AND (? IS NULL OR finalization_token = ?) + AND ( + subscription_effect_retry_at IS NULL + OR subscription_effect_retry_at <= ? + ) + AND ( + subscription_effect_state IS NULL + OR subscription_effect_state = 'pending' + OR ( + subscription_effect_state = 'processing' + AND subscription_effect_lease_at < ? + ) + )`, + detailJsonFromMailboxInboundSnapshot(next), + subscriptionEffectLease, + now, + now, + deliveryId, + mailboxInboundProvider, + input.expectedFinalizationToken ?? null, + input.expectedFinalizationToken ?? null, + now, + expiredBefore, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if ( + cursor.rowsWritten > 0 && + after?.subscriptionEffectLease === subscriptionEffectLease + ) { + return { status: 'claimed', delivery: after } + } + if ( + after?.subscriptionEffectState === 'complete' || + after?.subscriptionEffectState === 'dead-letter' + ) { + return { status: 'already-complete', delivery: after } + } + return { status: 'not-claimable', delivery: after } +} + +export function completeMailboxInboundSubscriptionEffect( + sql: SqlStorage, + input: { + deliveryId: string + subscriptionEffectLease: string + mode: 'complete' | 'suppressed' + suppressionReason?: string | null + now?: string + }, +): MailboxCompleteInboundSubscriptionEffectResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const subscriptionEffectLease = assertMailboxNonEmptyString( + input.subscriptionEffectLease, + 'subscriptionEffectLease', + ) + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current) return { status: 'lease-lost' } + if ( + current.subscriptionEffectState === 'complete' || + current.subscriptionEffectState === 'dead-letter' + ) { + return { status: 'already-complete', delivery: current } + } + if (current.subscriptionEffectLease !== subscriptionEffectLease) { + return { status: 'lease-lost' } + } + const extra = + input.mode === 'suppressed' + ? { + subscriptionEffectSuppressedAt: now, + ...(input.suppressionReason + ? { suppressionReason: input.suppressionReason } + : {}), + } + : undefined + const next: MailboxInboundDeliverySnapshot = { + ...current, + subscriptionEffectState: 'complete', + updatedAt: now, + } + delete next.subscriptionEffectLease + delete next.subscriptionEffectLeaseAt + delete next.subscriptionEffectRetryAt + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + subscription_effect_state = 'complete', + subscription_effect_lease = NULL, + subscription_effect_lease_at = NULL, + subscription_effect_retry_at = NULL, + needs_effect_reconcile = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND subscription_effect_lease = ?`, + detailJsonFromMailboxInboundSnapshot(next, extra), + needsMailboxEffectReconcile(next) ? 1 : 0, + now, + deliveryId, + mailboxInboundProvider, + subscriptionEffectLease, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (cursor.rowsWritten > 0 && after?.subscriptionEffectState === 'complete') { + return { + status: input.mode === 'suppressed' ? 'suppressed' : 'complete', + delivery: after, + } + } + if ( + after?.subscriptionEffectState === 'complete' || + after?.subscriptionEffectState === 'dead-letter' + ) { + return { status: 'already-complete', delivery: after } + } + return { status: 'lease-lost' } +} + +export function failMailboxInboundSubscriptionEffect( + sql: SqlStorage, + input: { + deliveryId: string + subscriptionEffectLease: string + error: string + now?: string + }, +): MailboxFailInboundSubscriptionEffectResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const subscriptionEffectLease = assertMailboxNonEmptyString( + input.subscriptionEffectLease, + 'subscriptionEffectLease', + ) + const error = assertMailboxNonEmptyString(input.error, 'error') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current || current.subscriptionEffectLease !== subscriptionEffectLease) { + return { status: 'lease-lost' } + } + const failure = resolveMailboxSubscriptionEffectFailure( + current.subscriptionEffectAttemptCount ?? 0, + ) + const next: MailboxInboundDeliverySnapshot = { + ...current, + subscriptionEffectState: failure.deadLettered ? 'dead-letter' : 'pending', + subscriptionEffectAttemptCount: failure.attemptCount, + subscriptionEffectLastError: error, + ...(failure.deadLettered + ? { subscriptionEffectDeadLetterAt: now } + : { + subscriptionEffectRetryAt: new Date( + Date.parse(now) + mailboxEffectRetryMs, + ).toISOString(), + }), + updatedAt: now, + } + delete next.subscriptionEffectLease + delete next.subscriptionEffectLeaseAt + if (failure.deadLettered) delete next.subscriptionEffectRetryAt + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + subscription_effect_state = ?, + subscription_effect_lease = NULL, + subscription_effect_lease_at = NULL, + subscription_effect_retry_at = ?, + subscription_effect_attempt_count = ?, + subscription_effect_dead_letter_at = ?, + subscription_effect_last_error = ?, + needs_effect_reconcile = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND subscription_effect_lease = ?`, + detailJsonFromMailboxInboundSnapshot(next), + next.subscriptionEffectState ?? 'pending', + next.subscriptionEffectRetryAt ?? null, + failure.attemptCount, + next.subscriptionEffectDeadLetterAt ?? null, + error, + needsMailboxEffectReconcile(next) ? 1 : 0, + now, + deliveryId, + mailboxInboundProvider, + subscriptionEffectLease, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (cursor.rowsWritten < 1 || !after) return { status: 'lease-lost' } + if (after.subscriptionEffectState === 'dead-letter') { + return { status: 'dead-letter', delivery: after } + } + if (after.subscriptionEffectState === 'pending') { + return { status: 'retry', delivery: after } + } + return { status: 'lease-lost' } +} + +export function listMailboxDueInboundEffectWork( + sql: SqlStorage, + input: { now?: string; limit?: number }, +): MailboxListDueInboundEffectWorkResult { + const now = normalizeMailboxInboundNow(input.now) + const limit = normalizeMailboxInboundLimit(input.limit) + const usageLeaseExpiredBefore = new Date( + Date.parse(now) - mailboxUsageEffectLeaseMs, + ).toISOString() + const subscriptionLeaseExpiredBefore = new Date( + Date.parse(now) - mailboxSubscriptionEffectLeaseMs, + ).toISOString() + const deliveries: Array = [] + for (const row of sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE provider = ? + AND event_type = 'received' + AND needs_effect_reconcile = 1 + AND fingerprint IS NOT NULL + AND ( + ( + usage_effect_recorded_at IS NULL + AND usage_effect_suppressed_at IS NULL + AND ( + usage_effect_retry_at IS NULL + OR usage_effect_retry_at <= ? + ) + AND ( + usage_effect_lease IS NULL + OR usage_effect_lease_at < ? + ) + ) + OR ( + ( + subscription_effect_state IS NULL + OR subscription_effect_state NOT IN ('complete', 'dead-letter') + ) + AND ( + subscription_effect_retry_at IS NULL + OR subscription_effect_retry_at <= ? + ) + AND ( + subscription_effect_state != 'processing' + OR subscription_effect_lease_at < ? + ) + ) + ) + ORDER BY COALESCE( + usage_effect_retry_at, + subscription_effect_retry_at, + created_at + ) ASC, id ASC + LIMIT ?`, + mailboxInboundProvider, + now, + usageLeaseExpiredBefore, + now, + subscriptionLeaseExpiredBefore, + limit, + ) + .toArray()) { + const snapshot = snapshotFromMailboxDeliveryEventRow( + mapMailboxDeliveryEventRow(row), + ) + if (snapshot) deliveries.push(snapshot) + } + return { deliveries } +} diff --git a/packages/worker/src/email/mailbox-inbound-ledger-shared.ts b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts new file mode 100644 index 0000000000..4689c7a83c --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts @@ -0,0 +1,375 @@ +/** + * Shared snapshot / detail helpers and constants for Mailbox inbound ledger + * CAS (step 2a). + */ + +import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' +import { + assertMailboxCanonicalIsoTimestamp, + mailboxNowIso, + type MailboxDeliveryEventRecord, + type MailboxInboundDeliveryState, + type MailboxSubscriptionEffectState, +} from './mailbox-types.ts' + +export const mailboxInboundProvider = 'cloudflare-email-routing' +export const mailboxInboundDedupeProvider = 'cloudflare-email-routing-dedupe' +export const mailboxInboundStorageLeaseMs = 5 * 60 * 1000 +export const mailboxInboundReconciliationRetryMs = 15 * 60 * 1000 +export const mailboxInboundDeliveryDedupeWindowMs = 48 * 60 * 60 * 1000 +export const mailboxStaleInboundDeliveryAgeMs = 48 * 60 * 60 * 1000 +export const mailboxUsageEffectLeaseMs = 5 * 60 * 1000 +export const mailboxSubscriptionEffectLeaseMs = 5 * 60 * 1000 +export const mailboxEffectRetryMs = 15 * 60 * 1000 +export const mailboxMaxSubscriptionEffectAttempts = 3 +export const mailboxInboundDueWorkDefaultLimit = 20 + +/** Inbound delivery identity + CAS fields (D1 `InboundDelivery` parity). */ +export type MailboxInboundDeliverySnapshot = { + fingerprint: string + deliveryId: string + messageId: string + threadId: string + rawMimeKey: string + inboxId: string + recipient: string + envelopeFrom: string + provider: string + quotaDay: string + dedupeExpiresAt: string + state: MailboxInboundDeliveryState + rejectionReason?: string + storageLease?: string + storageLeaseAt?: string + expectedAttachmentCount?: number + cleanupLease?: string + cleanupLeaseAt?: string + cleanupRetryAt?: string + finalizationToken?: string + reconcileAfter?: string + usageEffectRecordedAt?: string + usageEffectSuppressedAt?: string + usageStartedAt?: string + usageDurationMs?: number + usageMonth?: string + usageBytes?: number + usageEffectRetryAt?: string + usageEffectLease?: string + usageEffectLeaseAt?: string + subscriptionEffectState?: MailboxSubscriptionEffectState + subscriptionEffectLease?: string + subscriptionEffectLeaseAt?: string + subscriptionEffectRetryAt?: string + subscriptionEffectAttemptCount?: number + subscriptionEffectDeadLetterAt?: string + subscriptionEffectLastError?: string + createdAt: string + updatedAt: string +} + +export function optionalDetailString(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +export function optionalDetailNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +export function parseMailboxInboundDetail( + detailJson: string, +): Record { + try { + const parsed = JSON.parse(detailJson) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch { + // Fall through. + } + return {} +} + +function assignOpt( + target: T, + key: keyof MailboxInboundDeliverySnapshot, + value: string | number | undefined, +): void { + if (value !== undefined) { + ;(target as Record)[key as string] = value + } +} + +export function snapshotFromMailboxDeliveryEventRow( + row: MailboxDeliveryEventRecord, +): MailboxInboundDeliverySnapshot | null { + const detail = parseMailboxInboundDetail(row.detailJson) + const fingerprint = + row.fingerprint ?? optionalDetailString(detail['fingerprint']) + const messageId = + optionalDetailString(detail['messageId']) ?? row.messageId ?? undefined + const threadId = optionalDetailString(detail['threadId']) + const rawMimeKey = optionalDetailString(detail['rawMimeKey']) + const inboxId = + optionalDetailString(detail['inboxId']) ?? row.inboxId ?? undefined + const recipient = optionalDetailString(detail['recipient']) + const envelopeFrom = optionalDetailString(detail['envelopeFrom']) + const quotaDay = optionalDetailString(detail['quotaDay']) + const dedupeExpiresAt = + row.dedupeExpiresAt ?? optionalDetailString(detail['dedupeExpiresAt']) + if ( + !fingerprint || + !messageId || + !threadId || + !rawMimeKey || + !inboxId || + !recipient || + !envelopeFrom || + !quotaDay || + !dedupeExpiresAt + ) { + return null + } + const detailState = detail['state'] + const state: MailboxInboundDeliveryState = + row.state ?? + (detailState === 'storing' || + detailState === 'cleaning' || + detailState === 'received' || + detailState === 'rejected' || + detailState === 'orphan-cleaned' + ? detailState + : 'pending') + const detailSub = detail['subscriptionEffectState'] + const subscriptionEffectState = + row.subscriptionEffectState ?? + (detailSub === 'pending' || + detailSub === 'processing' || + detailSub === 'complete' || + detailSub === 'dead-letter' + ? detailSub + : undefined) + const snapshot: MailboxInboundDeliverySnapshot = { + fingerprint, + deliveryId: optionalDetailString(detail['deliveryId']) ?? row.id, + messageId, + threadId, + rawMimeKey, + inboxId, + recipient, + envelopeFrom, + provider: + optionalDetailString(detail['provider']) ?? + row.provider ?? + mailboxInboundProvider, + quotaDay, + dedupeExpiresAt, + state, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + } + assignOpt( + snapshot, + 'rejectionReason', + optionalDetailString(detail['rejectionReason']), + ) + assignOpt( + snapshot, + 'storageLease', + row.storageLease ?? optionalDetailString(detail['storageLease']), + ) + assignOpt( + snapshot, + 'storageLeaseAt', + row.storageLeaseAt ?? optionalDetailString(detail['storageLeaseAt']), + ) + assignOpt( + snapshot, + 'expectedAttachmentCount', + row.expectedAttachmentCount ?? + optionalDetailNumber(detail['expectedAttachmentCount']), + ) + assignOpt( + snapshot, + 'cleanupLease', + row.cleanupLease ?? optionalDetailString(detail['cleanupLease']), + ) + assignOpt( + snapshot, + 'cleanupLeaseAt', + row.cleanupLeaseAt ?? optionalDetailString(detail['cleanupLeaseAt']), + ) + assignOpt( + snapshot, + 'cleanupRetryAt', + row.cleanupRetryAt ?? optionalDetailString(detail['cleanupRetryAt']), + ) + assignOpt( + snapshot, + 'finalizationToken', + row.finalizationToken ?? optionalDetailString(detail['finalizationToken']), + ) + assignOpt( + snapshot, + 'reconcileAfter', + row.reconcileAfter ?? optionalDetailString(detail['reconcileAfter']), + ) + assignOpt( + snapshot, + 'usageEffectRecordedAt', + row.usageEffectRecordedAt ?? + optionalDetailString(detail['usageEffectRecordedAt']), + ) + assignOpt( + snapshot, + 'usageEffectSuppressedAt', + row.usageEffectSuppressedAt ?? + optionalDetailString(detail['usageEffectSuppressedAt']), + ) + assignOpt( + snapshot, + 'usageStartedAt', + row.usageStartedAt ?? optionalDetailString(detail['usageStartedAt']), + ) + assignOpt( + snapshot, + 'usageDurationMs', + row.usageDurationMs ?? optionalDetailNumber(detail['usageDurationMs']), + ) + assignOpt( + snapshot, + 'usageMonth', + row.usageMonth ?? optionalDetailString(detail['usageMonth']), + ) + assignOpt( + snapshot, + 'usageBytes', + row.usageBytes ?? optionalDetailNumber(detail['usageBytes']), + ) + assignOpt( + snapshot, + 'usageEffectRetryAt', + row.usageEffectRetryAt ?? + optionalDetailString(detail['usageEffectRetryAt']), + ) + assignOpt( + snapshot, + 'usageEffectLease', + row.usageEffectLease ?? optionalDetailString(detail['usageEffectLease']), + ) + assignOpt( + snapshot, + 'usageEffectLeaseAt', + row.usageEffectLeaseAt ?? + optionalDetailString(detail['usageEffectLeaseAt']), + ) + if (subscriptionEffectState) { + snapshot.subscriptionEffectState = subscriptionEffectState + } + assignOpt( + snapshot, + 'subscriptionEffectLease', + row.subscriptionEffectLease ?? + optionalDetailString(detail['subscriptionEffectLease']), + ) + assignOpt( + snapshot, + 'subscriptionEffectLeaseAt', + row.subscriptionEffectLeaseAt ?? + optionalDetailString(detail['subscriptionEffectLeaseAt']), + ) + assignOpt( + snapshot, + 'subscriptionEffectRetryAt', + row.subscriptionEffectRetryAt ?? + optionalDetailString(detail['subscriptionEffectRetryAt']), + ) + assignOpt( + snapshot, + 'subscriptionEffectAttemptCount', + row.subscriptionEffectAttemptCount ?? + optionalDetailNumber(detail['subscriptionEffectAttemptCount']), + ) + assignOpt( + snapshot, + 'subscriptionEffectDeadLetterAt', + row.subscriptionEffectDeadLetterAt ?? + optionalDetailString(detail['subscriptionEffectDeadLetterAt']), + ) + assignOpt( + snapshot, + 'subscriptionEffectLastError', + row.subscriptionEffectLastError ?? + optionalDetailString(detail['subscriptionEffectLastError']), + ) + return snapshot +} + +export function detailJsonFromMailboxInboundSnapshot( + snapshot: MailboxInboundDeliverySnapshot, + extra?: Record, +): string { + const { createdAt: _c, updatedAt: _u, ...rest } = snapshot + return JSON.stringify({ ...rest, ...extra }) +} + +export function readMailboxDeliveryEventRow( + sql: SqlStorage, + id: string, +): MailboxDeliveryEventRecord | null { + const row = sql + .exec>( + `SELECT * FROM email_delivery_events WHERE id = ? LIMIT 1`, + id, + ) + .toArray()[0] + return row ? mapMailboxDeliveryEventRow(row) : null +} + +export function readMailboxInboundDeliveryById( + sql: SqlStorage, + deliveryId: string, +): MailboxInboundDeliverySnapshot | null { + const row = readMailboxDeliveryEventRow(sql, deliveryId) + if (!row || row.provider !== mailboxInboundProvider) return null + return snapshotFromMailboxDeliveryEventRow(row) +} + +export function mailboxMessageExists(sql: SqlStorage, messageId: string) { + return ( + sql + .exec<{ ok: number }>( + `SELECT 1 AS ok FROM email_messages WHERE id = ? LIMIT 1`, + messageId, + ) + .toArray()[0] != null + ) +} + +export function needsMailboxEffectReconcile( + snapshot: Pick< + MailboxInboundDeliverySnapshot, + | 'usageEffectRecordedAt' + | 'usageEffectSuppressedAt' + | 'subscriptionEffectState' + >, +) { + const usageDone = + snapshot.usageEffectRecordedAt != null || + snapshot.usageEffectSuppressedAt != null + const subscriptionDone = + snapshot.subscriptionEffectState === 'complete' || + snapshot.subscriptionEffectState === 'dead-letter' + return !(usageDone && subscriptionDone) +} + +export function normalizeMailboxInboundNow(now?: string) { + if (now == null) return mailboxNowIso() + return assertMailboxCanonicalIsoTimestamp(now, 'now') +} + +export function normalizeMailboxInboundLimit(limit: number | undefined) { + if (typeof limit !== 'number' || !Number.isFinite(limit)) { + return mailboxInboundDueWorkDefaultLimit + } + return Math.min(Math.max(Math.trunc(limit), 1), 100) +} diff --git a/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts new file mode 100644 index 0000000000..f7d1aea52e --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts @@ -0,0 +1,61 @@ +import { expect, test } from 'vitest' +import { + resolveMailboxSubscriptionEffectFailure, + needsMailboxEffectReconcile, + mailboxInboundStorageLeaseMs, + mailboxMaxSubscriptionEffectAttempts, +} from './mailbox-inbound-ledger.ts' +import { resolveSubscriptionEffectFailure } from './inbound-effects.ts' + +test('subscription effect failure matrix matches D1 resolveSubscriptionEffectFailure', () => { + for (const attempt of [0, 1, 2, 3, 10]) { + expect(resolveMailboxSubscriptionEffectFailure(attempt)).toEqual( + resolveSubscriptionEffectFailure(attempt), + ) + } + expect(mailboxMaxSubscriptionEffectAttempts).toBe(3) + expect(resolveMailboxSubscriptionEffectFailure(2).deadLettered).toBe(true) + expect(resolveMailboxSubscriptionEffectFailure(1).deadLettered).toBe(false) +}) + +test('effect reconcile flag parity matches D1 needs_effect_reconcile cases', () => { + expect( + needsMailboxEffectReconcile({ + usageEffectRecordedAt: '2026-07-01T00:00:00.000Z', + subscriptionEffectState: 'complete', + }), + ).toBe(false) + expect( + needsMailboxEffectReconcile({ + usageEffectSuppressedAt: '2026-07-01T00:00:00.000Z', + subscriptionEffectState: 'dead-letter', + }), + ).toBe(false) + expect( + needsMailboxEffectReconcile({ + usageEffectRecordedAt: '2026-07-01T00:00:00.000Z', + subscriptionEffectState: 'pending', + }), + ).toBe(true) + expect( + needsMailboxEffectReconcile({ + subscriptionEffectState: 'complete', + }), + ).toBe(true) + expect(needsMailboxEffectReconcile({})).toBe(true) +}) + +test('storage lease takeover predicate matches D1 stale window', () => { + const now = Date.parse('2026-07-22T00:00:00.000Z') + const freshLeaseAt = new Date( + now - mailboxInboundStorageLeaseMs + 1, + ).toISOString() + const staleLeaseAt = new Date( + now - mailboxInboundStorageLeaseMs - 1, + ).toISOString() + const expiredBefore = new Date( + now - mailboxInboundStorageLeaseMs, + ).toISOString() + expect(freshLeaseAt < expiredBefore).toBe(false) + expect(staleLeaseAt < expiredBefore).toBe(true) +}) diff --git a/packages/worker/src/email/mailbox-inbound-ledger.ts b/packages/worker/src/email/mailbox-inbound-ledger.ts new file mode 100644 index 0000000000..9d41a6f6e4 --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-ledger.ts @@ -0,0 +1,807 @@ +/** + * Additive Mailbox USER inbound ledger CAS primitives (step 2a). + * + * Owner-bound SQLite helpers matching D1 USER transitions in + * `inbound-delivery.ts`. Effect lease CAS lives in + * `mailbox-inbound-effect-ledger.ts`. No external usage/subscription side + * effects. Mirror upsert RPCs remain the compatibility write path; these CAS + * RPCs are not live-wired (D1 stays authority). `system:email` stays on D1. + */ + +import { emailRawMimeKey } from './blob-keys.ts' +import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxNonEmptyString, + type MailboxInboundDeliveryState, +} from './mailbox-types.ts' +import { + detailJsonFromMailboxInboundSnapshot, + mailboxInboundDedupeProvider, + mailboxInboundProvider, + mailboxInboundReconciliationRetryMs, + mailboxInboundStorageLeaseMs, + mailboxMaxSubscriptionEffectAttempts, + mailboxMessageExists, + mailboxStaleInboundDeliveryAgeMs, + needsMailboxEffectReconcile, + normalizeMailboxInboundLimit, + normalizeMailboxInboundNow, + readMailboxDeliveryEventRow, + readMailboxInboundDeliveryById, + snapshotFromMailboxDeliveryEventRow, + type MailboxInboundDeliverySnapshot, +} from './mailbox-inbound-ledger-shared.ts' + +export { + detailJsonFromMailboxInboundSnapshot, + mailboxEffectRetryMs, + mailboxInboundDedupeProvider, + mailboxInboundDeliveryDedupeWindowMs, + mailboxInboundDueWorkDefaultLimit, + mailboxInboundProvider, + mailboxInboundReconciliationRetryMs, + mailboxInboundStorageLeaseMs, + mailboxMaxSubscriptionEffectAttempts, + mailboxMessageExists, + mailboxStaleInboundDeliveryAgeMs, + mailboxSubscriptionEffectLeaseMs, + mailboxUsageEffectLeaseMs, + needsMailboxEffectReconcile, + normalizeMailboxInboundLimit, + normalizeMailboxInboundNow, + readMailboxDeliveryEventRow, + readMailboxInboundDeliveryById, + snapshotFromMailboxDeliveryEventRow, + type MailboxInboundDeliverySnapshot, +} from './mailbox-inbound-ledger-shared.ts' + +/** Charged pending insert input (quota already accepted via UserMeter). */ +export type MailboxInboundDeliveryInsertInput = { + fingerprint: string + deliveryId: string + messageId: string + threadId: string + rawMimeKey: string + inboxId: string + recipient: string + envelopeFrom: string + provider?: string + quotaDay: string + dedupeExpiresAt: string + usageStartedAt?: string | null +} + +export type MailboxInsertChargedPendingInboundDeliveryResult = + | { status: 'inserted'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'existed'; delivery: MailboxInboundDeliverySnapshot } + +export type MailboxClaimInboundDeliveryStorageResult = + | { status: 'claimed'; delivery: MailboxInboundDeliverySnapshot } + | { + status: 'not-claimed' + delivery: MailboxInboundDeliverySnapshot | null + } + +export type MailboxReleaseInboundDeliveryStorageResult = + | { status: 'released'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'not-held' } + +export type MailboxMarkInboundDeliveryRejectedResult = + | { status: 'rejected'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'already-rejected'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'already-received'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'lease-lost' } + +export type MailboxMarkInboundDeliveryReceivedResult = + | { status: 'received'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'already-received'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'lease-lost' } + +export type MailboxDeferInboundDeliveryReconcileResult = + | { status: 'deferred'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'missing' } + | { status: 'not-applicable' } + +export type MailboxPruneExpiredInboundDedupeResult = { pruned: number } + +export type MailboxListDueStaleInboundDeliveriesResult = { + deliveries: Array +} + +export type MailboxInboundDeliveryLedgerRpc = { + getInboundDelivery: (input: { + ownerId: string + deliveryId: string + }) => Promise + getInboundDeliveryWindow: (input: { + ownerId: string + fingerprint: string + now?: string + }) => Promise + claimInboundDeliveryWindow: (input: { + ownerId: string + delivery: MailboxInboundDeliveryInsertInput + now?: string + }) => Promise + insertChargedPendingInboundDelivery: (input: { + ownerId: string + delivery: MailboxInboundDeliveryInsertInput + now?: string + }) => Promise + claimInboundDeliveryStorage: (input: { + ownerId: string + deliveryId: string + expectedAttachmentCount: number + usageStartedAt?: string | null + now?: string + }) => Promise + releaseInboundDeliveryStorage: (input: { + ownerId: string + deliveryId: string + storageLease: string + now?: string + }) => Promise + markInboundDeliveryRejected: (input: { + ownerId: string + deliveryId: string + reason: string + expectedStorageLease?: string | null + expectedState?: MailboxInboundDeliveryState + now?: string + }) => Promise + markInboundDeliveryReceived: (input: { + ownerId: string + deliveryId: string + storageLease: string + usageDurationMs: number + usageMonth: string + usageBytes: number + now?: string + }) => Promise + pruneExpiredInboundDedupePointers: (input: { + ownerId: string + now?: string + limit?: number + }) => Promise + deferInboundDeliveryReconciliation: (input: { + ownerId: string + deliveryId: string + now?: string + }) => Promise + listDueStaleInboundDeliveries: (input: { + ownerId: string + now?: string + limit?: number + }) => Promise +} + +export function mailboxInboundDedupePointerId(fingerprint: string) { + return `email-inbound-dedupe:${assertMailboxNonEmptyString(fingerprint, 'fingerprint')}` +} + +export function resolveMailboxSubscriptionEffectFailure(attemptCount: number) { + const nextAttempt = Math.max(0, Math.floor(attemptCount)) + 1 + return { + attemptCount: nextAttempt, + deadLettered: nextAttempt >= mailboxMaxSubscriptionEffectAttempts, + } +} + +function assertInsertInput( + ownerId: string, + delivery: MailboxInboundDeliveryInsertInput, +) { + const deliveryId = assertMailboxNonEmptyString( + delivery.deliveryId, + 'delivery.deliveryId', + ) + const messageId = assertMailboxNonEmptyString( + delivery.messageId, + 'delivery.messageId', + ) + const fingerprint = assertMailboxNonEmptyString( + delivery.fingerprint, + 'delivery.fingerprint', + ) + const threadId = assertMailboxNonEmptyString( + delivery.threadId, + 'delivery.threadId', + ) + const rawMimeKey = assertMailboxNonEmptyString( + delivery.rawMimeKey, + 'delivery.rawMimeKey', + ) + if (rawMimeKey !== emailRawMimeKey(ownerId, messageId)) { + throw new Error( + 'Mailbox inbound rawMimeKey must equal emailRawMimeKey(ownerId, messageId).', + ) + } + assertMailboxCanonicalIsoTimestamp( + delivery.dedupeExpiresAt, + 'delivery.dedupeExpiresAt', + ) + assertMailboxNonEmptyString(delivery.inboxId, 'delivery.inboxId') + assertMailboxNonEmptyString(delivery.recipient, 'delivery.recipient') + assertMailboxNonEmptyString(delivery.envelopeFrom, 'delivery.envelopeFrom') + assertMailboxNonEmptyString(delivery.quotaDay, 'delivery.quotaDay') + return { deliveryId, messageId, fingerprint, threadId, rawMimeKey } +} + +function pendingSnapshotFromInsert( + delivery: MailboxInboundDeliveryInsertInput, + validated: ReturnType, + provider: string, +): Omit { + return { + fingerprint: validated.fingerprint, + deliveryId: validated.deliveryId, + messageId: validated.messageId, + threadId: validated.threadId, + rawMimeKey: validated.rawMimeKey, + inboxId: delivery.inboxId, + recipient: delivery.recipient, + envelopeFrom: delivery.envelopeFrom, + provider, + quotaDay: delivery.quotaDay, + dedupeExpiresAt: delivery.dedupeExpiresAt, + state: 'pending', + ...(delivery.usageStartedAt + ? { usageStartedAt: delivery.usageStartedAt } + : {}), + } +} + +export function getMailboxInboundDelivery( + sql: SqlStorage, + deliveryId: string, +): MailboxInboundDeliverySnapshot | null { + return readMailboxInboundDeliveryById( + sql, + assertMailboxNonEmptyString(deliveryId, 'deliveryId'), + ) +} + +export function getMailboxInboundDeliveryWindow( + sql: SqlStorage, + input: { fingerprint: string; now?: string }, +): MailboxInboundDeliverySnapshot | null { + const now = normalizeMailboxInboundNow(input.now) + const pointerId = mailboxInboundDedupePointerId(input.fingerprint) + const row = sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE id = ? AND provider = ? AND dedupe_expires_at > ? + LIMIT 1`, + pointerId, + mailboxInboundDedupeProvider, + now, + ) + .toArray()[0] + return row + ? snapshotFromMailboxDeliveryEventRow(mapMailboxDeliveryEventRow(row)) + : null +} + +export function claimMailboxInboundDeliveryWindow( + sql: SqlStorage, + input: { + ownerId: string + delivery: MailboxInboundDeliveryInsertInput + now?: string + }, +): MailboxInboundDeliverySnapshot { + const now = normalizeMailboxInboundNow(input.now) + const ownerId = assertMailboxNonEmptyString(input.ownerId, 'ownerId') + const validated = assertInsertInput(ownerId, input.delivery) + const pointerId = mailboxInboundDedupePointerId(validated.fingerprint) + const base = pendingSnapshotFromInsert( + input.delivery, + validated, + input.delivery.provider ?? mailboxInboundProvider, + ) + // Pointer rows use the dedupe provider; deliveryId in detail still points + // at the charged delivery identity for window resolution. + const detail = detailJsonFromMailboxInboundSnapshot({ + ...base, + createdAt: now, + updatedAt: now, + }) + sql.exec( + `INSERT INTO email_delivery_events ( + id, message_id, inbox_id, event_type, provider, provider_event_id, + detail_json, needs_effect_reconcile, state, fingerprint, + dedupe_expires_at, usage_started_at, created_at, updated_at + ) VALUES (?, NULL, ?, 'receive_started', ?, ?, ?, 0, 'pending', ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + inbox_id = excluded.inbox_id, + detail_json = excluded.detail_json, + state = excluded.state, + fingerprint = excluded.fingerprint, + dedupe_expires_at = excluded.dedupe_expires_at, + usage_started_at = excluded.usage_started_at, + provider = excluded.provider, + provider_event_id = excluded.provider_event_id, + created_at = excluded.created_at, + updated_at = excluded.updated_at + WHERE email_delivery_events.dedupe_expires_at <= ?`, + pointerId, + input.delivery.inboxId, + mailboxInboundDedupeProvider, + pointerId, + detail, + validated.fingerprint, + input.delivery.dedupeExpiresAt, + input.delivery.usageStartedAt ?? null, + now, + now, + now, + ) + const row = readMailboxDeliveryEventRow(sql, pointerId) + const snapshot = row ? snapshotFromMailboxDeliveryEventRow(row) : null + if (!snapshot) { + throw new Error('Failed to resolve the inbound delivery dedupe window.') + } + return snapshot +} + +/** + * Persist a UserMeter-accepted pending delivery. Explicit inserted/existed + * matches the UserMeter consume replay shape. + */ +export function insertMailboxChargedPendingInboundDelivery( + sql: SqlStorage, + input: { + ownerId: string + delivery: MailboxInboundDeliveryInsertInput + now?: string + }, +): MailboxInsertChargedPendingInboundDeliveryResult { + const now = normalizeMailboxInboundNow(input.now) + const ownerId = assertMailboxNonEmptyString(input.ownerId, 'ownerId') + const validated = assertInsertInput(ownerId, input.delivery) + const existing = readMailboxInboundDeliveryById(sql, validated.deliveryId) + if (existing) return { status: 'existed', delivery: existing } + + const base = pendingSnapshotFromInsert( + input.delivery, + validated, + mailboxInboundProvider, + ) + const cursor = sql.exec( + `INSERT OR IGNORE INTO email_delivery_events ( + id, message_id, inbox_id, event_type, provider, provider_event_id, + detail_json, needs_effect_reconcile, state, fingerprint, + dedupe_expires_at, usage_started_at, created_at, updated_at + ) VALUES (?, NULL, ?, 'receive_started', ?, ?, ?, 0, 'pending', ?, ?, ?, ?, ?)`, + validated.deliveryId, + input.delivery.inboxId, + mailboxInboundProvider, + validated.deliveryId, + detailJsonFromMailboxInboundSnapshot({ + ...base, + createdAt: now, + updatedAt: now, + }), + validated.fingerprint, + input.delivery.dedupeExpiresAt, + input.delivery.usageStartedAt ?? null, + now, + now, + ) + const after = readMailboxInboundDeliveryById(sql, validated.deliveryId) + if (!after) { + throw new Error( + 'Inbound delivery charge was accepted but the delivery event was not persisted.', + ) + } + return { + status: cursor.rowsWritten > 0 ? 'inserted' : 'existed', + delivery: after, + } +} + +export function claimMailboxInboundDeliveryStorage( + sql: SqlStorage, + input: { + deliveryId: string + expectedAttachmentCount: number + usageStartedAt?: string | null + now?: string + }, +): MailboxClaimInboundDeliveryStorageResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const storageLease = crypto.randomUUID() + const expiredBefore = new Date( + Date.parse(now) - mailboxInboundStorageLeaseMs, + ).toISOString() + const expectedAttachmentCount = Math.max( + 0, + Math.floor(input.expectedAttachmentCount), + ) + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current) return { status: 'not-claimed', delivery: null } + + const canClaim = + current.state === 'pending' || + (current.state === 'storing' && + current.storageLeaseAt != null && + current.storageLeaseAt < expiredBefore) || + (current.state === 'received' && + !mailboxMessageExists(sql, current.messageId)) + if (!canClaim) return { status: 'not-claimed', delivery: current } + + const usageStartedAt = current.usageStartedAt ?? input.usageStartedAt ?? null + const next: MailboxInboundDeliverySnapshot = { + ...current, + state: 'storing', + storageLease, + storageLeaseAt: now, + expectedAttachmentCount, + ...(usageStartedAt ? { usageStartedAt } : {}), + updatedAt: now, + } + delete next.finalizationToken + sql.exec( + `UPDATE email_delivery_events + SET event_type = 'receive_started', + message_id = NULL, + detail_json = ?, + state = 'storing', + storage_lease = ?, + storage_lease_at = ?, + expected_attachment_count = ?, + usage_started_at = COALESCE(usage_started_at, ?), + finalization_token = NULL, + updated_at = ? + WHERE id = ? + AND provider = ? + AND ( + state = 'pending' + OR (state = 'storing' AND storage_lease_at < ?) + OR ( + state = 'received' + AND NOT EXISTS (SELECT 1 FROM email_messages WHERE id = ?) + ) + )`, + detailJsonFromMailboxInboundSnapshot(next), + storageLease, + now, + expectedAttachmentCount, + usageStartedAt, + now, + deliveryId, + mailboxInboundProvider, + expiredBefore, + current.messageId, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (after?.storageLease === storageLease && after.state === 'storing') { + return { status: 'claimed', delivery: after } + } + return { status: 'not-claimed', delivery: after } +} + +export function releaseMailboxInboundDeliveryStorage( + sql: SqlStorage, + input: { + deliveryId: string + storageLease: string + now?: string + }, +): MailboxReleaseInboundDeliveryStorageResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const storageLease = assertMailboxNonEmptyString( + input.storageLease, + 'storageLease', + ) + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if ( + !current || + current.state !== 'storing' || + current.storageLease !== storageLease + ) { + return { status: 'not-held' } + } + const next: MailboxInboundDeliverySnapshot = { + ...current, + state: 'pending', + updatedAt: now, + } + delete next.storageLease + delete next.storageLeaseAt + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + state = 'pending', + storage_lease = NULL, + storage_lease_at = NULL, + updated_at = ? + WHERE id = ? + AND provider = ? + AND state = 'storing' + AND storage_lease = ?`, + detailJsonFromMailboxInboundSnapshot(next), + now, + deliveryId, + mailboxInboundProvider, + storageLease, + ) + if (cursor.rowsWritten < 1) return { status: 'not-held' } + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (!after) return { status: 'not-held' } + return { status: 'released', delivery: after } +} + +export function markMailboxInboundDeliveryRejected( + sql: SqlStorage, + input: { + deliveryId: string + reason: string + expectedStorageLease?: string | null + expectedState?: MailboxInboundDeliveryState + now?: string + }, +): MailboxMarkInboundDeliveryRejectedResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const reason = assertMailboxNonEmptyString(input.reason, 'reason') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current) return { status: 'lease-lost' } + if (current.state === 'rejected') { + return { status: 'already-rejected', delivery: current } + } + if (current.state === 'received') { + return mailboxMessageExists(sql, current.messageId) + ? { status: 'already-received', delivery: current } + : { status: 'lease-lost' } + } + const expectedLease = input.expectedStorageLease ?? null + const expectedState = input.expectedState ?? current.state + if ( + current.state !== expectedState || + (current.storageLease ?? null) !== expectedLease + ) { + return { status: 'lease-lost' } + } + const next: MailboxInboundDeliverySnapshot = { + ...current, + state: 'rejected', + rejectionReason: reason, + updatedAt: now, + } + const cursor = sql.exec( + `UPDATE email_delivery_events + SET event_type = 'rejected', + detail_json = ?, + state = 'rejected', + updated_at = ? + WHERE id = ? + AND provider = ? + AND event_type = 'receive_started' + AND state = ? + AND ( + (? IS NULL AND storage_lease IS NULL) + OR storage_lease = ? + )`, + detailJsonFromMailboxInboundSnapshot(next), + now, + deliveryId, + mailboxInboundProvider, + expectedState, + expectedLease, + expectedLease, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (cursor.rowsWritten > 0 && after?.state === 'rejected') { + return { status: 'rejected', delivery: after } + } + if (after?.state === 'rejected') { + return { status: 'already-rejected', delivery: after } + } + if ( + after?.state === 'received' && + mailboxMessageExists(sql, after.messageId) + ) { + return { status: 'already-received', delivery: after } + } + return { status: 'lease-lost' } +} + +export function markMailboxInboundDeliveryReceived( + sql: SqlStorage, + input: { + deliveryId: string + storageLease: string + usageDurationMs: number + usageMonth: string + usageBytes: number + now?: string + }, +): MailboxMarkInboundDeliveryReceivedResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const storageLease = assertMailboxNonEmptyString( + input.storageLease, + 'storageLease', + ) + const usageMonth = assertMailboxNonEmptyString(input.usageMonth, 'usageMonth') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current) return { status: 'lease-lost' } + if (current.state === 'received') { + return { status: 'already-received', delivery: current } + } + if (current.state !== 'storing' || current.storageLease !== storageLease) { + return { status: 'lease-lost' } + } + const usageDurationMs = Math.max(0, Math.round(input.usageDurationMs)) + const usageBytes = Math.max(0, Math.round(input.usageBytes)) + const next: MailboxInboundDeliverySnapshot = { + ...current, + state: 'received', + finalizationToken: storageLease, + subscriptionEffectState: current.subscriptionEffectState ?? 'pending', + usageDurationMs: current.usageDurationMs ?? usageDurationMs, + usageMonth: current.usageMonth ?? usageMonth, + usageBytes: current.usageBytes ?? usageBytes, + updatedAt: now, + } + delete next.storageLease + delete next.storageLeaseAt + const cursor = sql.exec( + `UPDATE email_delivery_events + SET message_id = ?, + event_type = 'received', + detail_json = ?, + state = 'received', + storage_lease = NULL, + storage_lease_at = NULL, + finalization_token = ?, + subscription_effect_state = ?, + usage_duration_ms = ?, + usage_month = ?, + usage_bytes = ?, + usage_effect_recorded_at = ?, + needs_effect_reconcile = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND state = 'storing' + AND storage_lease = ?`, + current.messageId, + detailJsonFromMailboxInboundSnapshot(next), + storageLease, + next.subscriptionEffectState ?? 'pending', + next.usageDurationMs ?? usageDurationMs, + next.usageMonth ?? usageMonth, + next.usageBytes ?? usageBytes, + next.usageEffectRecordedAt ?? null, + needsMailboxEffectReconcile(next) ? 1 : 0, + now, + deliveryId, + mailboxInboundProvider, + storageLease, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (cursor.rowsWritten > 0 && after?.state === 'received') { + return { status: 'received', delivery: after } + } + if (after?.state === 'received') { + return { status: 'already-received', delivery: after } + } + return { status: 'lease-lost' } +} + +export function pruneMailboxExpiredInboundDedupePointers( + sql: SqlStorage, + input: { now?: string; limit?: number }, +): MailboxPruneExpiredInboundDedupeResult { + const now = normalizeMailboxInboundNow(input.now) + const limit = normalizeMailboxInboundLimit(input.limit) + const ids = sql + .exec<{ id: string }>( + `SELECT id FROM email_delivery_events + WHERE provider = ? AND dedupe_expires_at <= ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + mailboxInboundDedupeProvider, + now, + limit, + ) + .toArray() + .map((row) => String(row.id)) + let pruned = 0 + for (const id of ids) { + const cursor = sql.exec( + `DELETE FROM email_delivery_events + WHERE id = ? AND provider = ? AND dedupe_expires_at <= ?`, + id, + mailboxInboundDedupeProvider, + now, + ) + pruned += cursor.rowsWritten + } + return { pruned } +} + +export function deferMailboxInboundDeliveryReconciliation( + sql: SqlStorage, + input: { deliveryId: string; now?: string }, +): MailboxDeferInboundDeliveryReconcileResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current) return { status: 'missing' } + if (current.state === 'received' || current.state === 'rejected') { + return { status: 'not-applicable' } + } + const reconcileAfter = new Date( + Date.parse(now) + mailboxInboundReconciliationRetryMs, + ).toISOString() + const next: MailboxInboundDeliverySnapshot = { + ...current, + reconcileAfter, + updatedAt: now, + } + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, reconcile_after = ?, updated_at = ? + WHERE id = ? AND provider = ? AND event_type = 'receive_started'`, + detailJsonFromMailboxInboundSnapshot(next), + reconcileAfter, + now, + deliveryId, + mailboxInboundProvider, + ) + if (cursor.rowsWritten < 1) return { status: 'not-applicable' } + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if (!after) return { status: 'missing' } + return { status: 'deferred', delivery: after } +} + +export function listMailboxDueStaleInboundDeliveries( + sql: SqlStorage, + input: { now?: string; limit?: number }, +): MailboxListDueStaleInboundDeliveriesResult { + const now = normalizeMailboxInboundNow(input.now) + const limit = normalizeMailboxInboundLimit(input.limit) + const cutoff = new Date( + Date.parse(now) - mailboxStaleInboundDeliveryAgeMs, + ).toISOString() + const leaseExpiredBefore = new Date( + Date.parse(now) - mailboxInboundStorageLeaseMs, + ).toISOString() + const deliveries: Array = [] + for (const row of sql + .exec>( + `SELECT * FROM email_delivery_events + WHERE provider = ? + AND event_type = 'receive_started' + AND created_at < ? + AND (reconcile_after IS NULL OR reconcile_after <= ?) + AND ( + state = 'pending' + OR (state = 'storing' AND storage_lease_at < ?) + OR (state = 'cleaning' AND cleanup_lease_at < ?) + OR (state = 'orphan-cleaned' AND cleanup_retry_at <= ?) + ) + ORDER BY created_at ASC, id ASC + LIMIT ?`, + mailboxInboundProvider, + cutoff, + now, + leaseExpiredBefore, + leaseExpiredBefore, + now, + limit, + ) + .toArray()) { + const snapshot = snapshotFromMailboxDeliveryEventRow( + mapMailboxDeliveryEventRow(row), + ) + if (snapshot) deliveries.push(snapshot) + } + return { deliveries } +} diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts new file mode 100644 index 0000000000..8f52b95253 --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -0,0 +1,619 @@ +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { emailRawMimeKey } from './blob-keys.ts' +import { Mailbox } from './mailbox-do.ts' +import { + mailboxInboundDedupeProvider, + mailboxInboundProvider, + mailboxInboundStorageLeaseMs, + type MailboxInboundDeliveryInsertInput, +} from './mailbox-inbound-ledger.ts' +import { initializeMailboxSchema } from './mailbox-schema.ts' +import { + mailboxMetaSchemaVersionKey, + mailboxSchemaVersion, +} from './mailbox-types.ts' +import { + assertMailboxThrows, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' + +function insertInput( + ownerId: string, + overrides?: Partial, +): MailboxInboundDeliveryInsertInput { + const deliveryId = + overrides?.deliveryId ?? `email-inbound-delivery:${crypto.randomUUID()}` + const messageId = + overrides?.messageId ?? `email-inbound-message:${crypto.randomUUID()}` + const fingerprint = + overrides?.fingerprint ?? crypto.randomUUID().replace(/-/g, '') + return { + fingerprint, + deliveryId, + messageId, + threadId: + overrides?.threadId ?? `email-inbound-thread:${crypto.randomUUID()}`, + rawMimeKey: emailRawMimeKey(ownerId, messageId), + inboxId: overrides?.inboxId ?? `inbox-${crypto.randomUUID()}`, + recipient: overrides?.recipient ?? 'owner@example.com', + envelopeFrom: overrides?.envelopeFrom ?? 'sender@example.com', + quotaDay: overrides?.quotaDay ?? '2026-07-22', + dedupeExpiresAt: overrides?.dedupeExpiresAt ?? '2026-07-24T00:00:00.000Z', + usageStartedAt: overrides?.usageStartedAt, + provider: overrides?.provider, + } +} + +test('Mailbox inbound ledger CAS covers USER transition matrix without live flip', async () => { + silenceIncidentalRuntimeWarnings() + const ownerA = uniqueUserId('ledger-a') + const ownerB = uniqueUserId('ledger-b') + const mailboxA = rpcFor(ownerA) + const mailboxB = rpcFor(ownerB) + const now = '2026-07-22T00:00:00.000Z' + + // Schema v2 indexes present after cold init. + await runInDurableObject( + stubFor(ownerA), + async (_instance: Mailbox, state) => { + expect(mailboxSchemaVersion).toBe(2) + const version = state.storage.sql + .exec<{ value: number }>( + `SELECT value FROM mailbox_meta WHERE key = ?`, + mailboxMetaSchemaVersionKey, + ) + .toArray()[0]?.value + expect(Number(version)).toBe(mailboxSchemaVersion) + const indexes = state.storage.sql + .exec<{ name: string }>( + `SELECT name FROM sqlite_master + WHERE type = 'index' + AND name IN ( + 'idx_email_delivery_events_reconcile_after', + 'idx_email_delivery_events_usage_effect_retry', + 'idx_email_delivery_events_subscription_effect_retry', + 'idx_email_delivery_events_stale_state', + 'idx_email_delivery_events_dedupe_provider_expires', + 'idx_email_delivery_events_state_created', + 'idx_email_delivery_events_dedupe_expires' + ) + ORDER BY name ASC`, + ) + .toArray() + .map((row) => row.name) + expect(indexes).toContain('idx_email_delivery_events_reconcile_after') + expect(indexes).toContain('idx_email_delivery_events_usage_effect_retry') + expect(indexes).toContain( + 'idx_email_delivery_events_subscription_effect_retry', + ) + expect(indexes).toContain('idx_email_delivery_events_stale_state') + expect(indexes).toContain( + 'idx_email_delivery_events_dedupe_provider_expires', + ) + }, + ) + + const delivery = insertInput(ownerA, { + fingerprint: 'fp-concurrent-dedupe', + deliveryId: 'email-inbound-delivery:concurrent', + messageId: 'email-inbound-message:concurrent', + threadId: 'email-inbound-thread:concurrent', + }) + + // Concurrent dedupe window claim / rewrite. + const window1 = await mailboxA.claimInboundDeliveryWindow({ + ownerId: ownerA, + delivery, + now, + }) + expect(window1.deliveryId).toBe(delivery.deliveryId) + expect(window1.fingerprint).toBe(delivery.fingerprint) + const windowAgain = await mailboxA.claimInboundDeliveryWindow({ + ownerId: ownerA, + delivery: { + ...delivery, + deliveryId: 'email-inbound-delivery:rewritten', + messageId: 'email-inbound-message:rewritten', + threadId: 'email-inbound-thread:rewritten', + rawMimeKey: emailRawMimeKey(ownerA, 'email-inbound-message:rewritten'), + }, + now: '2026-07-22T01:00:00.000Z', + }) + // Active window is not rewritten. + expect(windowAgain.deliveryId).toBe(delivery.deliveryId) + const activeWindow = await mailboxA.getInboundDeliveryWindow({ + ownerId: ownerA, + fingerprint: delivery.fingerprint, + now, + }) + expect(activeWindow?.deliveryId).toBe(delivery.deliveryId) + + // UserMeter-following insert replay shape: inserted then existed. + const inserted = await mailboxA.insertChargedPendingInboundDelivery({ + ownerId: ownerA, + delivery, + now, + }) + expect(inserted.status).toBe('inserted') + const replay = await mailboxA.insertChargedPendingInboundDelivery({ + ownerId: ownerA, + delivery, + now: '2026-07-22T00:00:01.000Z', + }) + expect(replay.status).toBe('existed') + expect(replay.delivery.deliveryId).toBe(delivery.deliveryId) + + // Storage lease claim / stale release / finalize vs reject. + const claim = await mailboxA.claimInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(claim.status).toBe('claimed') + if (claim.status !== 'claimed') throw new Error('expected claim') + const lease = claim.delivery.storageLease + expect(lease).toBeTruthy() + + const staleRelease = await mailboxA.releaseInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + storageLease: 'wrong-lease', + now, + }) + expect(staleRelease.status).toBe('not-held') + + const released = await mailboxA.releaseInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + storageLease: lease!, + now: '2026-07-22T00:00:02.000Z', + }) + expect(released.status).toBe('released') + + const reclaimAt = '2026-07-22T00:00:03.000Z' + const reclaim = await mailboxA.claimInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now: reclaimAt, + }) + expect(reclaim.status).toBe('claimed') + if (reclaim.status !== 'claimed') throw new Error('expected reclaim') + + // Stale lease cannot finalize after takeover. + const staleLease = reclaim.delivery.storageLease + const takeoverNow = new Date( + Date.parse(reclaimAt) + mailboxInboundStorageLeaseMs + 1_000, + ).toISOString() + const takeover = await mailboxA.claimInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now: takeoverNow, + }) + expect(takeover.status).toBe('claimed') + if (takeover.status !== 'claimed') throw new Error('expected takeover') + expect(takeover.delivery.storageLease).not.toBe(staleLease) + + const staleFinalize = await mailboxA.markInboundDeliveryReceived({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + storageLease: staleLease!, + usageDurationMs: 10, + usageMonth: '2026-07', + usageBytes: 32, + now: takeoverNow, + }) + expect(staleFinalize.status).toBe('lease-lost') + + const received = await mailboxA.markInboundDeliveryReceived({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + storageLease: takeover.delivery.storageLease!, + usageDurationMs: 25, + usageMonth: '2026-07', + usageBytes: 64, + now: takeoverNow, + }) + expect(received.status).toBe('received') + if (received.status !== 'received') throw new Error('expected received') + expect(received.delivery.state).toBe('received') + expect(received.delivery.finalizationToken).toBe( + takeover.delivery.storageLease, + ) + expect(received.delivery.subscriptionEffectState).toBe('pending') + + const rejectAfterReceived = await mailboxA.markInboundDeliveryRejected({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + reason: 'too-late', + now: takeoverNow, + }) + // No message row → lease-lost (D1 would throw); with message would be already-received. + expect(rejectAfterReceived.status).toBe('lease-lost') + + // Rejected path on a separate delivery. + const rejectedDelivery = insertInput(ownerA, { + fingerprint: 'fp-rejected', + deliveryId: 'email-inbound-delivery:rejected', + messageId: 'email-inbound-message:rejected', + threadId: 'email-inbound-thread:rejected', + }) + await mailboxA.insertChargedPendingInboundDelivery({ + ownerId: ownerA, + delivery: rejectedDelivery, + now, + }) + const rejected = await mailboxA.markInboundDeliveryRejected({ + ownerId: ownerA, + deliveryId: rejectedDelivery.deliveryId, + reason: 'policy', + expectedStorageLease: null, + expectedState: 'pending', + now, + }) + expect(rejected.status).toBe('rejected') + + // Usage + subscription effect exactly-once leases / retry / dead-letter. + const usageClaim = await mailboxA.claimInboundUsageEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: received.delivery.finalizationToken, + now: takeoverNow, + }) + expect(usageClaim.status).toBe('claimed') + if (usageClaim.status !== 'claimed') throw new Error('expected usage claim') + const usageBusy = await mailboxA.claimInboundUsageEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + now: takeoverNow, + }) + expect(usageBusy.status).toBe('not-claimable') + const usageDone = await mailboxA.completeInboundUsageEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + usageEffectLease: usageClaim.delivery.usageEffectLease!, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 64, + usageDurationMs: 25, + now: takeoverNow, + }) + expect(usageDone.status).toBe('recorded') + const usageReplay = await mailboxA.claimInboundUsageEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + now: takeoverNow, + }) + expect(usageReplay.status).toBe('already-complete') + + const subClaim = await mailboxA.claimInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: received.delivery.finalizationToken, + now: takeoverNow, + }) + expect(subClaim.status).toBe('claimed') + if (subClaim.status !== 'claimed') throw new Error('expected sub claim') + const fail1 = await mailboxA.failInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: subClaim.delivery.subscriptionEffectLease!, + error: 'transient-1', + now: takeoverNow, + }) + expect(fail1.status).toBe('retry') + + const retryAt = + fail1.status === 'retry' + ? fail1.delivery.subscriptionEffectRetryAt! + : takeoverNow + const subClaim2 = await mailboxA.claimInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + now: retryAt, + }) + expect(subClaim2.status).toBe('claimed') + if (subClaim2.status !== 'claimed') throw new Error('expected sub claim 2') + const fail2 = await mailboxA.failInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: subClaim2.delivery.subscriptionEffectLease!, + error: 'transient-2', + now: retryAt, + }) + expect(fail2.status).toBe('retry') + const retryAt2 = + fail2.status === 'retry' + ? fail2.delivery.subscriptionEffectRetryAt! + : retryAt + const subClaim3 = await mailboxA.claimInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + now: retryAt2, + }) + expect(subClaim3.status).toBe('claimed') + if (subClaim3.status !== 'claimed') throw new Error('expected sub claim 3') + const dead = await mailboxA.failInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: subClaim3.delivery.subscriptionEffectLease!, + error: 'final', + now: retryAt2, + }) + expect(dead.status).toBe('dead-letter') + + // Suppression path on a fresh received delivery. + const suppressDelivery = insertInput(ownerA, { + fingerprint: 'fp-suppress', + deliveryId: 'email-inbound-delivery:suppress', + messageId: 'email-inbound-message:suppress', + threadId: 'email-inbound-thread:suppress', + }) + await mailboxA.insertChargedPendingInboundDelivery({ + ownerId: ownerA, + delivery: suppressDelivery, + now, + }) + const suppressClaim = await mailboxA.claimInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: suppressDelivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(suppressClaim.status).toBe('claimed') + if (suppressClaim.status !== 'claimed') throw new Error('expected claim') + const suppressReceived = await mailboxA.markInboundDeliveryReceived({ + ownerId: ownerA, + deliveryId: suppressDelivery.deliveryId, + storageLease: suppressClaim.delivery.storageLease!, + usageDurationMs: 1, + usageMonth: '2026-07', + usageBytes: 1, + now, + }) + expect(suppressReceived.status).toBe('received') + const usageSuppressClaim = await mailboxA.claimInboundUsageEffect({ + ownerId: ownerA, + deliveryId: suppressDelivery.deliveryId, + now, + }) + expect(usageSuppressClaim.status).toBe('claimed') + if (usageSuppressClaim.status !== 'claimed') { + throw new Error('expected usage suppress claim') + } + expect( + ( + await mailboxA.completeInboundUsageEffect({ + ownerId: ownerA, + deliveryId: suppressDelivery.deliveryId, + usageEffectLease: usageSuppressClaim.delivery.usageEffectLease!, + mode: 'suppressed', + usageMonth: '2026-07', + usageBytes: 1, + usageDurationMs: 1, + now, + }) + ).status, + ).toBe('suppressed') + const subSuppressClaim = await mailboxA.claimInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: suppressDelivery.deliveryId, + now, + }) + expect(subSuppressClaim.status).toBe('claimed') + if (subSuppressClaim.status !== 'claimed') { + throw new Error('expected sub suppress claim') + } + expect( + ( + await mailboxA.completeInboundSubscriptionEffect({ + ownerId: ownerA, + deliveryId: suppressDelivery.deliveryId, + subscriptionEffectLease: + subSuppressClaim.delivery.subscriptionEffectLease!, + mode: 'suppressed', + suppressionReason: 'quarantine', + now, + }) + ).status, + ).toBe('suppressed') + + // Due listing + owner isolation. + const stalePending = insertInput(ownerA, { + fingerprint: 'fp-stale', + deliveryId: 'email-inbound-delivery:stale', + messageId: 'email-inbound-message:stale', + threadId: 'email-inbound-thread:stale', + }) + await mailboxA.insertChargedPendingInboundDelivery({ + ownerId: ownerA, + delivery: stalePending, + now: '2026-07-19T00:00:00.000Z', + }) + const dueStale = await mailboxA.listDueStaleInboundDeliveries({ + ownerId: ownerA, + now: '2026-07-22T00:00:00.000Z', + limit: 50, + }) + expect( + dueStale.deliveries.some((d) => d.deliveryId === stalePending.deliveryId), + ).toBe(true) + + const foreign = insertInput(ownerB, { + fingerprint: 'fp-b', + deliveryId: 'email-inbound-delivery:owner-b', + messageId: 'email-inbound-message:owner-b', + threadId: 'email-inbound-thread:owner-b', + }) + await mailboxB.insertChargedPendingInboundDelivery({ + ownerId: ownerB, + delivery: foreign, + now, + }) + expect( + await mailboxA.getInboundDelivery({ + ownerId: ownerA, + deliveryId: foreign.deliveryId, + }), + ).toBeNull() + await runInDurableObject(stubFor(ownerA), async (instance: Mailbox) => { + await assertMailboxThrows(/ownerId mismatch/, () => + instance.insertChargedPendingInboundDelivery({ + ownerId: ownerB, + delivery: foreign, + now, + }), + ) + }) + + // Dedupe prune. + await runInDurableObject(stubFor(ownerA), async (_instance, state) => { + state.storage.sql.exec( + `UPDATE email_delivery_events + SET dedupe_expires_at = ?, updated_at = ? + WHERE provider = ?`, + '2026-07-21T00:00:00.000Z', + now, + mailboxInboundDedupeProvider, + ) + }) + const pruned = await mailboxA.pruneExpiredInboundDedupePointers({ + ownerId: ownerA, + now: '2026-07-22T00:00:00.000Z', + limit: 50, + }) + expect(pruned.pruned).toBeGreaterThan(0) + + // Defer reconcile. + const deferred = await mailboxA.deferInboundDeliveryReconciliation({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + now, + }) + expect(deferred.status).toBe('deferred') + + // Mirror upsert compatibility still works alongside ledger rows. + const mirror = await mailboxA.upsertDeliveryEvent({ + ownerId: ownerA, + event: { + id: 'mirror-compat-event', + messageId: null, + inboxId: null, + eventType: 'failed', + provider: 'kody', + providerMessageId: null, + providerEventId: 'mirror-compat-event', + detailJson: '{}', + needsEffectReconcile: false, + state: null, + fingerprint: null, + storageLease: null, + storageLeaseAt: null, + cleanupLease: null, + cleanupLeaseAt: null, + cleanupRetryAt: null, + expectedAttachmentCount: null, + finalizationToken: null, + reconcileAfter: null, + dedupeExpiresAt: null, + usageEffectRecordedAt: null, + usageEffectSuppressedAt: null, + usageStartedAt: null, + usageMonth: null, + usageBytes: null, + usageDurationMs: null, + usageEffectRetryAt: null, + usageEffectLease: null, + usageEffectLeaseAt: null, + subscriptionEffectState: null, + subscriptionEffectLease: null, + subscriptionEffectLeaseAt: null, + subscriptionEffectRetryAt: null, + subscriptionEffectAttemptCount: null, + subscriptionEffectDeadLetterAt: null, + subscriptionEffectLastError: null, + createdAt: now, + updatedAt: now, + }, + }) + expect(mirror.accepted).toBe(true) + + // Export / purge / retention coexistence. + const exported = await mailboxA.exportMailbox({ pageSize: 50 }) + expect( + exported.rows.some( + (row) => + row.kind === 'delivery_event' && + row.row.provider === mailboxInboundProvider, + ), + ).toBe(true) + await mailboxA.purge() + expect( + await mailboxA.getInboundDelivery({ + ownerId: ownerA, + deliveryId: delivery.deliveryId, + }), + ).toBeNull() + // Re-init after purge still at schema v2. + await runInDurableObject(stubFor(ownerA), async (_instance, state) => { + const version = state.storage.sql + .exec<{ value: number }>( + `SELECT value FROM mailbox_meta WHERE key = ?`, + mailboxMetaSchemaVersionKey, + ) + .toArray()[0]?.value + expect(Number(version)).toBe(mailboxSchemaVersion) + }) +}) + +test('Mailbox inbound ledger warm-migrates v1 schema indexes to v2', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('warm-v2') + const stub = stubFor(ownerId) + await runInDurableObject(stub, async (instance: Mailbox, state) => { + expect(instance).toBeInstanceOf(Mailbox) + // Simulate a warm v1 object that somehow lacks v2 indexes. + state.storage.sql.exec( + `UPDATE mailbox_meta SET value = 1 WHERE key = ?`, + mailboxMetaSchemaVersionKey, + ) + state.storage.sql.exec( + `DROP INDEX IF EXISTS idx_email_delivery_events_reconcile_after`, + ) + state.storage.sql.exec( + `DROP INDEX IF EXISTS idx_email_delivery_events_usage_effect_retry`, + ) + state.storage.sql.exec( + `DROP INDEX IF EXISTS idx_email_delivery_events_subscription_effect_retry`, + ) + state.storage.sql.exec( + `DROP INDEX IF EXISTS idx_email_delivery_events_stale_state`, + ) + state.storage.sql.exec( + `DROP INDEX IF EXISTS idx_email_delivery_events_dedupe_provider_expires`, + ) + // Re-run schema init (same path as constructor / purge). + initializeMailboxSchema(state.storage) + const version = state.storage.sql + .exec<{ value: number }>( + `SELECT value FROM mailbox_meta WHERE key = ?`, + mailboxMetaSchemaVersionKey, + ) + .toArray()[0]?.value + expect(Number(version)).toBe(mailboxSchemaVersion) + const hasReconcile = state.storage.sql + .exec<{ name: string }>( + `SELECT name FROM sqlite_master + WHERE type = 'index' + AND name = 'idx_email_delivery_events_reconcile_after'`, + ) + .toArray() + expect(hasReconcile.length).toBe(1) + }) +}) diff --git a/packages/worker/src/email/mailbox-schema.ts b/packages/worker/src/email/mailbox-schema.ts index 77a8c8948e..7fd8ee741d 100644 --- a/packages/worker/src/email/mailbox-schema.ts +++ b/packages/worker/src/email/mailbox-schema.ts @@ -30,8 +30,11 @@ export function setMailboxMeta( } /** - * Schema v1 — coherent ship shape (this DO class has never been deployed). - * No warm ALTER migrations; bump `mailboxSchemaVersion` only with a real plan. + * Schema bootstrap + warm migrations. + * - v1: base tables/indexes (deployed). + * - v2: additive inbound ledger query indexes (state/reconcile/retry/dedupe). + * Warm objects re-run CREATE IF NOT EXISTS then apply any version-gated + * index additions; never destructive. */ export function initializeMailboxSchema(storage: DurableObjectStorage) { const sql = storage.sql @@ -300,5 +303,41 @@ export function initializeMailboxSchema(storage: DurableObjectStorage) { WHERE usage_effect_recorded_at IS NOT NULL`, ) + // v2 — inbound ledger CAS due-work indexes (safe on warm v1 objects). + if (installedVersion == null || installedVersion < 2) { + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_reconcile_after + ON email_delivery_events(reconcile_after ASC, created_at ASC, id ASC) + WHERE reconcile_after IS NOT NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_usage_effect_retry + ON email_delivery_events(usage_effect_retry_at ASC, id ASC) + WHERE needs_effect_reconcile = 1 + AND usage_effect_recorded_at IS NULL + AND usage_effect_suppressed_at IS NULL`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_subscription_effect_retry + ON email_delivery_events(subscription_effect_retry_at ASC, id ASC) + WHERE needs_effect_reconcile = 1 + AND ( + subscription_effect_state IS NULL + OR subscription_effect_state NOT IN ('complete', 'dead-letter') + )`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_stale_state + ON email_delivery_events(state, created_at ASC, id ASC) + WHERE state IN ('pending', 'storing', 'cleaning', 'orphan-cleaned')`, + ) + sql.exec( + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_dedupe_provider_expires + ON email_delivery_events(provider, dedupe_expires_at ASC, id ASC) + WHERE provider = 'cloudflare-email-routing-dedupe' + AND dedupe_expires_at IS NOT NULL`, + ) + } + setMailboxMeta(storage, mailboxMetaSchemaVersionKey, mailboxSchemaVersion) } diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 835d9236bb..62964e5f46 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -105,6 +105,11 @@ function buildMailboxMessageFilterClauses(input: { /** * SQLite write/query helpers for one Mailbox DO. No alarm / R2 side effects. + * + * Additive step 2a USER inbound ledger/effect CAS helpers live in + * `mailbox-inbound-ledger.ts` / `mailbox-inbound-effect-ledger.ts` (wired as + * owner-bound DO RPCs; not live-called — D1 remains authority). Mirror upsert + * paths below stay the compatibility write API. */ export class MailboxStore { private readonly storage: DurableObjectStorage diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 1549310751..4a8a7f1cf8 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -10,6 +10,8 @@ import { type EmailDirection, type EmailProcessingStatus, } from './types.ts' +import { type MailboxInboundDeliveryLedgerRpc } from './mailbox-inbound-ledger.ts' +import { type MailboxInboundEffectLedgerRpc } from './mailbox-inbound-effect-ledger.ts' /** * Mailbox Durable Object wire types, constants, and boundary validators. @@ -30,8 +32,11 @@ export const mailboxRetentionRetryDelayMs = 60 * 60 * 1000 export const mailboxRetentionContinuationDelayMs = 1_000 export const mailboxBlobDeleteMaxKeys = 1000 -/** Bump when initializeSchema's DDL set changes; warm objects skip DDL. */ -export const mailboxSchemaVersion = 1 +/** + * Bump when initializeSchema's DDL set changes. Warm objects run additive + * migrations (CREATE INDEX IF NOT EXISTS / guarded ALTER) until they match. + */ +export const mailboxSchemaVersion = 2 export const mailboxMetaSchemaVersionKey = 'schema_version' export const mailboxExportThreadCursorPrefix = 'thread:' @@ -473,7 +478,12 @@ export type MailboxUpsertDeliveryEventsResult = { results: Array } -export type MailboxRpc = { +/** + * Mirror / read / retention / purge surface. Inbound ledger CAS RPCs + * (additive step 2a) are intersected below — not live-wired; D1 remains + * authority. + */ +type MailboxCoreRpc = { mirrorMessage: (input: { /** * Explicit owner binding. DO name is not introspectable at runtime, so @@ -489,6 +499,10 @@ export type MailboxRpc = { */ attachments?: Array }) => Promise<{ ok: true; accepted: boolean }> + /** + * Compatibility mirror upsert for complete delivery-event snapshots. + * Remains available during step 2a; CAS ledger RPCs are additive. + */ upsertDeliveryEvent: (input: { ownerId: string event: MailboxDeliveryEventInput @@ -581,6 +595,11 @@ export type MailboxRpc = { purge: () => Promise<{ ok: true }> } +export type MailboxInboundLedgerRpc = MailboxInboundDeliveryLedgerRpc & + MailboxInboundEffectLedgerRpc + +export type MailboxRpc = MailboxCoreRpc & MailboxInboundLedgerRpc + export function mailboxNowIso() { return new Date().toISOString() } From 3be4121f88efc9e9f6d721fbc3360490162f1d83 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 13:52:32 +0000 Subject: [PATCH 47/62] test(email): allow replay watermark wall time Co-authored-by: Kent C. Dodds --- packages/worker/src/email/mailbox-reconcile.node.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/email/mailbox-reconcile.node.test.ts b/packages/worker/src/email/mailbox-reconcile.node.test.ts index 981ac2def5..505cc22c0c 100644 --- a/packages/worker/src/email/mailbox-reconcile.node.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.node.test.ts @@ -348,7 +348,9 @@ test('mailbox parity multi-run cursor progress, isolation, soak, errors, and cou }) const afterSoak = await readParityRow(db, ownerA) expect(afterSoak?.matchingSince).toBe(rematchNow.toISOString()) - expect(afterSoak?.contentWatermarkAt).toBe(soakNow.toISOString()) + expect( + Date.parse(afterSoak?.contentWatermarkAt ?? '') >= soakNow.getTime(), + ).toBe(true) expect(afterSoak?.mismatchCount).toBe(0) consoleWarn.mockClear() From 6580fb0131efebb211c49234a5f8987278d650a9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 14:07:55 +0000 Subject: [PATCH 48/62] fix(email): fence inbound effects by finalization Co-authored-by: Kent C. Dodds --- packages/worker/src/email/mailbox-do.ts | 3 + .../email/mailbox-inbound-effect-ledger.ts | 52 ++++- .../src/email/mailbox-inbound-ledger.ts | 25 ++ .../mailbox-inbound-ledger.workers.test.ts | 218 +++++++++++++++++- 4 files changed, 296 insertions(+), 2 deletions(-) diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index f26aa4f1a7..0bea8aec63 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -706,6 +706,7 @@ class MailboxBase extends DurableObject implements MailboxRpc { ownerId: string deliveryId: string usageEffectLease: string + expectedFinalizationToken: string mode: 'recorded' | 'suppressed' usageMonth: string usageBytes: number @@ -743,6 +744,7 @@ class MailboxBase extends DurableObject implements MailboxRpc { ownerId: string deliveryId: string subscriptionEffectLease: string + expectedFinalizationToken: string mode: 'complete' | 'suppressed' suppressionReason?: string | null now?: string @@ -764,6 +766,7 @@ class MailboxBase extends DurableObject implements MailboxRpc { ownerId: string deliveryId: string subscriptionEffectLease: string + expectedFinalizationToken: string error: string now?: string }) { diff --git a/packages/worker/src/email/mailbox-inbound-effect-ledger.ts b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts index d205f8671e..115404ccbf 100644 --- a/packages/worker/src/email/mailbox-inbound-effect-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts @@ -66,6 +66,8 @@ export type MailboxInboundEffectLedgerRpc = { ownerId: string deliveryId: string usageEffectLease: string + /** Must match the delivery's current finalization token (D1 parity). */ + expectedFinalizationToken: string mode: 'recorded' | 'suppressed' usageMonth: string usageBytes: number @@ -82,6 +84,8 @@ export type MailboxInboundEffectLedgerRpc = { ownerId: string deliveryId: string subscriptionEffectLease: string + /** Must match the delivery's current finalization token (D1 parity). */ + expectedFinalizationToken: string mode: 'complete' | 'suppressed' suppressionReason?: string | null now?: string @@ -90,6 +94,8 @@ export type MailboxInboundEffectLedgerRpc = { ownerId: string deliveryId: string subscriptionEffectLease: string + /** Must match the delivery's current finalization token (D1 parity). */ + expectedFinalizationToken: string error: string now?: string }) => Promise @@ -187,6 +193,7 @@ export function completeMailboxInboundUsageEffect( input: { deliveryId: string usageEffectLease: string + expectedFinalizationToken: string mode: 'recorded' | 'suppressed' usageMonth: string usageBytes: number @@ -200,9 +207,19 @@ export function completeMailboxInboundUsageEffect( input.usageEffectLease, 'usageEffectLease', ) + const expectedFinalizationToken = assertMailboxNonEmptyString( + input.expectedFinalizationToken, + 'expectedFinalizationToken', + ) const usageMonth = assertMailboxNonEmptyString(input.usageMonth, 'usageMonth') const current = readMailboxInboundDeliveryById(sql, deliveryId) if (!current) return { status: 'lease-lost' } + if ( + current.state !== 'received' || + current.finalizationToken !== expectedFinalizationToken + ) { + return { status: 'lease-lost' } + } if (current.usageEffectRecordedAt || current.usageEffectSuppressedAt) { return { status: 'already-complete', delivery: current } } @@ -239,6 +256,9 @@ export function completeMailboxInboundUsageEffect( updated_at = ? WHERE id = ? AND provider = ? + AND event_type = 'received' + AND state = 'received' + AND finalization_token = ? AND usage_effect_lease = ? AND usage_effect_recorded_at IS NULL AND usage_effect_suppressed_at IS NULL`, @@ -252,6 +272,7 @@ export function completeMailboxInboundUsageEffect( now, deliveryId, mailboxInboundProvider, + expectedFinalizationToken, usageEffectLease, ) const after = readMailboxInboundDeliveryById(sql, deliveryId) @@ -382,6 +403,7 @@ export function completeMailboxInboundSubscriptionEffect( input: { deliveryId: string subscriptionEffectLease: string + expectedFinalizationToken: string mode: 'complete' | 'suppressed' suppressionReason?: string | null now?: string @@ -393,8 +415,18 @@ export function completeMailboxInboundSubscriptionEffect( input.subscriptionEffectLease, 'subscriptionEffectLease', ) + const expectedFinalizationToken = assertMailboxNonEmptyString( + input.expectedFinalizationToken, + 'expectedFinalizationToken', + ) const current = readMailboxInboundDeliveryById(sql, deliveryId) if (!current) return { status: 'lease-lost' } + if ( + current.state !== 'received' || + current.finalizationToken !== expectedFinalizationToken + ) { + return { status: 'lease-lost' } + } if ( current.subscriptionEffectState === 'complete' || current.subscriptionEffectState === 'dead-letter' @@ -432,12 +464,16 @@ export function completeMailboxInboundSubscriptionEffect( updated_at = ? WHERE id = ? AND provider = ? + AND event_type = 'received' + AND state = 'received' + AND finalization_token = ? AND subscription_effect_lease = ?`, detailJsonFromMailboxInboundSnapshot(next, extra), needsMailboxEffectReconcile(next) ? 1 : 0, now, deliveryId, mailboxInboundProvider, + expectedFinalizationToken, subscriptionEffectLease, ) const after = readMailboxInboundDeliveryById(sql, deliveryId) @@ -461,6 +497,7 @@ export function failMailboxInboundSubscriptionEffect( input: { deliveryId: string subscriptionEffectLease: string + expectedFinalizationToken: string error: string now?: string }, @@ -471,9 +508,18 @@ export function failMailboxInboundSubscriptionEffect( input.subscriptionEffectLease, 'subscriptionEffectLease', ) + const expectedFinalizationToken = assertMailboxNonEmptyString( + input.expectedFinalizationToken, + 'expectedFinalizationToken', + ) const error = assertMailboxNonEmptyString(input.error, 'error') const current = readMailboxInboundDeliveryById(sql, deliveryId) - if (!current || current.subscriptionEffectLease !== subscriptionEffectLease) { + if ( + !current || + current.state !== 'received' || + current.finalizationToken !== expectedFinalizationToken || + current.subscriptionEffectLease !== subscriptionEffectLease + ) { return { status: 'lease-lost' } } const failure = resolveMailboxSubscriptionEffectFailure( @@ -510,6 +556,9 @@ export function failMailboxInboundSubscriptionEffect( updated_at = ? WHERE id = ? AND provider = ? + AND event_type = 'received' + AND state = 'received' + AND finalization_token = ? AND subscription_effect_lease = ?`, detailJsonFromMailboxInboundSnapshot(next), next.subscriptionEffectState ?? 'pending', @@ -521,6 +570,7 @@ export function failMailboxInboundSubscriptionEffect( now, deliveryId, mailboxInboundProvider, + expectedFinalizationToken, subscriptionEffectLease, ) const after = readMailboxInboundDeliveryById(sql, deliveryId) diff --git a/packages/worker/src/email/mailbox-inbound-ledger.ts b/packages/worker/src/email/mailbox-inbound-ledger.ts index 9d41a6f6e4..4ea13cebd2 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.ts @@ -433,6 +433,12 @@ export function claimMailboxInboundDeliveryStorage( if (!canClaim) return { status: 'not-claimed', delivery: current } const usageStartedAt = current.usageStartedAt ?? input.usageStartedAt ?? null + /** + * Storage reclaim invalidates prior finalization + in-flight effect leases. + * D1 `json_set` only patches storage fields (stale lease keys can linger in + * `detail_json`); promoted Mailbox columns must clear explicitly so a stale + * effect worker cannot complete against a later re-finalization. + */ const next: MailboxInboundDeliverySnapshot = { ...current, state: 'storing', @@ -441,8 +447,17 @@ export function claimMailboxInboundDeliveryStorage( expectedAttachmentCount, ...(usageStartedAt ? { usageStartedAt } : {}), updatedAt: now, + ...(current.subscriptionEffectState === 'processing' + ? { subscriptionEffectState: 'pending' as const } + : {}), } delete next.finalizationToken + delete next.usageEffectLease + delete next.usageEffectLeaseAt + delete next.usageEffectRetryAt + delete next.subscriptionEffectLease + delete next.subscriptionEffectLeaseAt + delete next.subscriptionEffectRetryAt sql.exec( `UPDATE email_delivery_events SET event_type = 'receive_started', @@ -454,6 +469,16 @@ export function claimMailboxInboundDeliveryStorage( expected_attachment_count = ?, usage_started_at = COALESCE(usage_started_at, ?), finalization_token = NULL, + usage_effect_lease = NULL, + usage_effect_lease_at = NULL, + usage_effect_retry_at = NULL, + subscription_effect_lease = NULL, + subscription_effect_lease_at = NULL, + subscription_effect_retry_at = NULL, + subscription_effect_state = CASE + WHEN subscription_effect_state = 'processing' THEN 'pending' + ELSE subscription_effect_state + END, updated_at = ? WHERE id = ? AND provider = ? diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts index 8f52b95253..7346639a72 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -274,10 +274,12 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip now: takeoverNow, }) expect(usageBusy.status).toBe('not-claimable') + const finalizationToken = received.delivery.finalizationToken! const usageDone = await mailboxA.completeInboundUsageEffect({ ownerId: ownerA, deliveryId: delivery.deliveryId, usageEffectLease: usageClaim.delivery.usageEffectLease!, + expectedFinalizationToken: finalizationToken, mode: 'recorded', usageMonth: '2026-07', usageBytes: 64, @@ -295,7 +297,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip const subClaim = await mailboxA.claimInboundSubscriptionEffect({ ownerId: ownerA, deliveryId: delivery.deliveryId, - expectedFinalizationToken: received.delivery.finalizationToken, + expectedFinalizationToken: finalizationToken, now: takeoverNow, }) expect(subClaim.status).toBe('claimed') @@ -304,6 +306,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip ownerId: ownerA, deliveryId: delivery.deliveryId, subscriptionEffectLease: subClaim.delivery.subscriptionEffectLease!, + expectedFinalizationToken: finalizationToken, error: 'transient-1', now: takeoverNow, }) @@ -316,6 +319,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip const subClaim2 = await mailboxA.claimInboundSubscriptionEffect({ ownerId: ownerA, deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, now: retryAt, }) expect(subClaim2.status).toBe('claimed') @@ -324,6 +328,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip ownerId: ownerA, deliveryId: delivery.deliveryId, subscriptionEffectLease: subClaim2.delivery.subscriptionEffectLease!, + expectedFinalizationToken: finalizationToken, error: 'transient-2', now: retryAt, }) @@ -335,6 +340,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip const subClaim3 = await mailboxA.claimInboundSubscriptionEffect({ ownerId: ownerA, deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, now: retryAt2, }) expect(subClaim3.status).toBe('claimed') @@ -343,6 +349,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip ownerId: ownerA, deliveryId: delivery.deliveryId, subscriptionEffectLease: subClaim3.delivery.subscriptionEffectLease!, + expectedFinalizationToken: finalizationToken, error: 'final', now: retryAt2, }) @@ -387,12 +394,17 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip if (usageSuppressClaim.status !== 'claimed') { throw new Error('expected usage suppress claim') } + const suppressToken = + suppressReceived.status === 'received' + ? suppressReceived.delivery.finalizationToken! + : '' expect( ( await mailboxA.completeInboundUsageEffect({ ownerId: ownerA, deliveryId: suppressDelivery.deliveryId, usageEffectLease: usageSuppressClaim.delivery.usageEffectLease!, + expectedFinalizationToken: suppressToken, mode: 'suppressed', usageMonth: '2026-07', usageBytes: 1, @@ -404,6 +416,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip const subSuppressClaim = await mailboxA.claimInboundSubscriptionEffect({ ownerId: ownerA, deliveryId: suppressDelivery.deliveryId, + expectedFinalizationToken: suppressToken, now, }) expect(subSuppressClaim.status).toBe('claimed') @@ -417,6 +430,7 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip deliveryId: suppressDelivery.deliveryId, subscriptionEffectLease: subSuppressClaim.delivery.subscriptionEffectLease!, + expectedFinalizationToken: suppressToken, mode: 'suppressed', suppressionReason: 'quarantine', now, @@ -572,6 +586,208 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip }) }) +test('stale effect workers cannot complete or fail after storage reclaim re-finalization', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('reclaim-fence') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-reclaim-fence', + deliveryId: 'email-inbound-delivery:reclaim-fence', + messageId: 'email-inbound-message:reclaim-fence', + threadId: 'email-inbound-thread:reclaim-fence', + }) + + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + const firstClaim = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(firstClaim.status).toBe('claimed') + if (firstClaim.status !== 'claimed') throw new Error('expected first claim') + const firstReceived = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: firstClaim.delivery.storageLease!, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: 8, + now, + }) + expect(firstReceived.status).toBe('received') + if (firstReceived.status !== 'received') throw new Error('expected received') + const staleToken = firstReceived.delivery.finalizationToken! + expect(staleToken).toBeTruthy() + + const staleUsageClaim = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: staleToken, + now, + }) + expect(staleUsageClaim.status).toBe('claimed') + if (staleUsageClaim.status !== 'claimed') { + throw new Error('expected stale usage claim') + } + const staleUsageLease = staleUsageClaim.delivery.usageEffectLease! + const staleSubClaim = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: staleToken, + now, + }) + expect(staleSubClaim.status).toBe('claimed') + if (staleSubClaim.status !== 'claimed') { + throw new Error('expected stale subscription claim') + } + const staleSubLease = staleSubClaim.delivery.subscriptionEffectLease! + + // Reclaim storage (no message row) — clears finalization + effect leases. + const reclaim = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now: '2026-07-22T00:01:00.000Z', + }) + expect(reclaim.status).toBe('claimed') + if (reclaim.status !== 'claimed') throw new Error('expected reclaim') + expect(reclaim.delivery.finalizationToken).toBeUndefined() + expect(reclaim.delivery.usageEffectLease).toBeUndefined() + expect(reclaim.delivery.subscriptionEffectLease).toBeUndefined() + expect(reclaim.delivery.subscriptionEffectState).toBe('pending') + + const secondReceived = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: reclaim.delivery.storageLease!, + usageDurationMs: 9, + usageMonth: '2026-07', + usageBytes: 16, + now: '2026-07-22T00:01:01.000Z', + }) + expect(secondReceived.status).toBe('received') + if (secondReceived.status !== 'received') { + throw new Error('expected second received') + } + const freshToken = secondReceived.delivery.finalizationToken! + expect(freshToken).toBeTruthy() + expect(freshToken).not.toBe(staleToken) + + // Stale workers with prior token/leases cannot complete or fail. + expect( + ( + await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + usageEffectLease: staleUsageLease, + expectedFinalizationToken: staleToken, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 8, + usageDurationMs: 5, + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + expect( + ( + await mailbox.completeInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: staleSubLease, + expectedFinalizationToken: staleToken, + mode: 'complete', + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + expect( + ( + await mailbox.failInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: staleSubLease, + expectedFinalizationToken: staleToken, + error: 'stale-fail', + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + // Stale lease + fresh token still loses (lease cleared on reclaim). + expect( + ( + await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + usageEffectLease: staleUsageLease, + expectedFinalizationToken: freshToken, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 8, + usageDurationMs: 5, + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + + // Fresh worker with the new finalization token can claim and complete. + const freshUsageClaim = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: freshToken, + now: '2026-07-22T00:01:03.000Z', + }) + expect(freshUsageClaim.status).toBe('claimed') + if (freshUsageClaim.status !== 'claimed') { + throw new Error('expected fresh usage claim') + } + expect( + ( + await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + usageEffectLease: freshUsageClaim.delivery.usageEffectLease!, + expectedFinalizationToken: freshToken, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 16, + usageDurationMs: 9, + now: '2026-07-22T00:01:03.000Z', + }) + ).status, + ).toBe('recorded') + + const freshSubClaim = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: freshToken, + now: '2026-07-22T00:01:04.000Z', + }) + expect(freshSubClaim.status).toBe('claimed') + if (freshSubClaim.status !== 'claimed') { + throw new Error('expected fresh subscription claim') + } + expect( + ( + await mailbox.completeInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: + freshSubClaim.delivery.subscriptionEffectLease!, + expectedFinalizationToken: freshToken, + mode: 'complete', + now: '2026-07-22T00:01:04.000Z', + }) + ).status, + ).toBe('complete') +}) + test('Mailbox inbound ledger warm-migrates v1 schema indexes to v2', async () => { silenceIncidentalRuntimeWarnings() const ownerId = uniqueUserId('warm-v2') From 1e14bb0b9966e8237556769328739da28e93887a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 14:18:53 +0000 Subject: [PATCH 49/62] docs(email): document effect finalization fence Co-authored-by: Kent C. Dodds --- docs/contributing/architecture/data-storage.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 4ba19272e9..3a5138935e 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -719,12 +719,18 @@ mirror D1 USER transitions from `inbound-delivery.ts` / `inbound-effects.ts` pending with explicit `inserted`/`existed`; claim/release storage lease; mark rejected/received with lease/finalization CAS; prune dedupe; defer reconcile; claim/complete usage effect; claim/complete/fail subscription effect with -retry/dead-letter/ suppression; list due stale/effect work). Mutations keep -promoted columns and `detail_json` in sync and set canonical `updated_at`. The -DO does **not** perform external usage recording or subscription dispatch. -Mirror `upsertDeliveryEvent` / `upsertDeliveryEvents` remain the compatibility -write API. **These CAS RPCs are not live-wired** — D1 stays sole write authority -for inbound ledger/effects. `system:email` stays on D1 and is never written into +retry/dead-letter/suppression; list due stale/effect work). Mutations keep +promoted columns and `detail_json` in sync and set canonical `updated_at`. +Usage/subscription complete and subscription fail require an exact +`expectedFinalizationToken` match (`event_type`/`state` = `received`); mismatch +is `lease-lost`. Storage claim clears finalization plus in-flight effect +leases/retry (and resets `processing` → `pending`) so reclaim/re-finalization +cannot be completed by a stale effect worker — stronger than D1's `json_set` +patch, required because Mailbox promotes those lease columns. The DO does +**not** perform external usage recording or subscription dispatch. Mirror +`upsertDeliveryEvent` / `upsertDeliveryEvents` remain the compatibility write +API. **These CAS RPCs are not live-wired** — D1 stays sole write authority for +inbound ledger/effects. `system:email` stays on D1 and is never written into per-user Mailbox objects. **Mirror write contract:** `mirrorMessage`, `upsertDeliveryEvent`, and From 40904d990550b626ed53554abb67e96b54eff54e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 14:29:54 +0000 Subject: [PATCH 50/62] fix(email): admit legacy inbound effect leases Co-authored-by: Kent C. Dodds --- .../email/mailbox-inbound-effect-ledger.ts | 31 +- .../email/mailbox-inbound-ledger-shared.ts | 11 + .../email/mailbox-inbound-ledger.node.test.ts | 23 ++ .../src/email/mailbox-inbound-ledger.ts | 21 +- .../mailbox-inbound-ledger.workers.test.ts | 309 +++++++++++++++++- packages/worker/src/email/mailbox-schema.ts | 2 + 6 files changed, 372 insertions(+), 25 deletions(-) diff --git a/packages/worker/src/email/mailbox-inbound-effect-ledger.ts b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts index 115404ccbf..913940b231 100644 --- a/packages/worker/src/email/mailbox-inbound-effect-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts @@ -10,6 +10,7 @@ import { assertMailboxNonEmptyString } from './mailbox-types.ts' import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' import { + assertMailboxInboundNonNegativeFiniteNumber, detailJsonFromMailboxInboundSnapshot, mailboxEffectRetryMs, mailboxInboundProvider, @@ -138,6 +139,8 @@ export function claimMailboxInboundUsageEffect( const expiredBefore = new Date( Date.parse(now) - mailboxUsageEffectLeaseMs, ).toISOString() + // Fresh lease only when lease+timestamp are both present and unexpired. + // NULL lease_at (legacy/imported) is treated as expired/claimable. if ( current.usageEffectLease != null && current.usageEffectLeaseAt != null && @@ -165,7 +168,11 @@ export function claimMailboxInboundUsageEffect( AND usage_effect_recorded_at IS NULL AND usage_effect_suppressed_at IS NULL AND (usage_effect_retry_at IS NULL OR usage_effect_retry_at <= ?) - AND (usage_effect_lease IS NULL OR usage_effect_lease_at < ?) + AND ( + usage_effect_lease IS NULL + OR usage_effect_lease_at IS NULL + OR usage_effect_lease_at < ? + ) AND (? IS NULL OR finalization_token = ?)`, detailJsonFromMailboxInboundSnapshot(next), usageEffectLease, @@ -226,8 +233,15 @@ export function completeMailboxInboundUsageEffect( if (current.usageEffectLease !== usageEffectLease) { return { status: 'lease-lost' } } - const usageBytes = Math.max(0, Math.round(input.usageBytes)) - const usageDurationMs = Math.max(0, Math.round(input.usageDurationMs)) + const usageBytes = Math.round( + assertMailboxInboundNonNegativeFiniteNumber(input.usageBytes, 'usageBytes'), + ) + const usageDurationMs = Math.round( + assertMailboxInboundNonNegativeFiniteNumber( + input.usageDurationMs, + 'usageDurationMs', + ), + ) const next: MailboxInboundDeliverySnapshot = { ...current, usageMonth, @@ -332,6 +346,7 @@ export function claimMailboxInboundSubscriptionEffect( const expiredBefore = new Date( Date.parse(now) - mailboxSubscriptionEffectLeaseMs, ).toISOString() + // processing + NULL lease_at (legacy/imported) is claimable/takeover. if ( current.subscriptionEffectState === 'processing' && current.subscriptionEffectLeaseAt != null && @@ -368,7 +383,10 @@ export function claimMailboxInboundSubscriptionEffect( OR subscription_effect_state = 'pending' OR ( subscription_effect_state = 'processing' - AND subscription_effect_lease_at < ? + AND ( + subscription_effect_lease_at IS NULL + OR subscription_effect_lease_at < ? + ) ) )`, detailJsonFromMailboxInboundSnapshot(next), @@ -614,6 +632,7 @@ export function listMailboxDueInboundEffectWork( ) AND ( usage_effect_lease IS NULL + OR usage_effect_lease_at IS NULL OR usage_effect_lease_at < ? ) ) @@ -627,7 +646,9 @@ export function listMailboxDueInboundEffectWork( OR subscription_effect_retry_at <= ? ) AND ( - subscription_effect_state != 'processing' + subscription_effect_state IS NULL + OR subscription_effect_state != 'processing' + OR subscription_effect_lease_at IS NULL OR subscription_effect_lease_at < ? ) ) diff --git a/packages/worker/src/email/mailbox-inbound-ledger-shared.ts b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts index 4689c7a83c..2ce1d0cf2a 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger-shared.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts @@ -373,3 +373,14 @@ export function normalizeMailboxInboundLimit(limit: number | undefined) { } return Math.min(Math.max(Math.trunc(limit), 1), 100) } + +/** Non-negative finite number; rejects NaN/Infinity before floor/round. */ +export function assertMailboxInboundNonNegativeFiniteNumber( + value: unknown, + label: string, +): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`Mailbox ${label} must be a finite number.`) + } + return Math.max(0, value) +} diff --git a/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts index f7d1aea52e..45e8bd0cfa 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest' import { + assertMailboxInboundNonNegativeFiniteNumber, resolveMailboxSubscriptionEffectFailure, needsMailboxEffectReconcile, mailboxInboundStorageLeaseMs, @@ -59,3 +60,25 @@ test('storage lease takeover predicate matches D1 stale window', () => { expect(freshLeaseAt < expiredBefore).toBe(false) expect(staleLeaseAt < expiredBefore).toBe(true) }) + +test('assertMailboxInboundNonNegativeFiniteNumber rejects NaN and Infinity', () => { + for (const value of [ + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]) { + for (const label of [ + 'expectedAttachmentCount', + 'usageDurationMs', + 'usageBytes', + ] as const) { + expect(() => + assertMailboxInboundNonNegativeFiniteNumber(value, label), + ).toThrow(new RegExp(`${label} must be a finite number`)) + } + } + expect(assertMailboxInboundNonNegativeFiniteNumber(0, 'usageBytes')).toBe(0) + expect( + assertMailboxInboundNonNegativeFiniteNumber(12.5, 'usageDurationMs'), + ).toBe(12.5) +}) diff --git a/packages/worker/src/email/mailbox-inbound-ledger.ts b/packages/worker/src/email/mailbox-inbound-ledger.ts index 4ea13cebd2..1f843311e5 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.ts @@ -16,6 +16,7 @@ import { type MailboxInboundDeliveryState, } from './mailbox-types.ts' import { + assertMailboxInboundNonNegativeFiniteNumber, detailJsonFromMailboxInboundSnapshot, mailboxInboundDedupeProvider, mailboxInboundProvider, @@ -34,6 +35,7 @@ import { } from './mailbox-inbound-ledger-shared.ts' export { + assertMailboxInboundNonNegativeFiniteNumber, detailJsonFromMailboxInboundSnapshot, mailboxEffectRetryMs, mailboxInboundDedupeProvider, @@ -416,9 +418,11 @@ export function claimMailboxInboundDeliveryStorage( const expiredBefore = new Date( Date.parse(now) - mailboxInboundStorageLeaseMs, ).toISOString() - const expectedAttachmentCount = Math.max( - 0, - Math.floor(input.expectedAttachmentCount), + const expectedAttachmentCount = Math.floor( + assertMailboxInboundNonNegativeFiniteNumber( + input.expectedAttachmentCount, + 'expectedAttachmentCount', + ), ) const current = readMailboxInboundDeliveryById(sql, deliveryId) if (!current) return { status: 'not-claimed', delivery: null } @@ -661,8 +665,15 @@ export function markMailboxInboundDeliveryReceived( if (current.state !== 'storing' || current.storageLease !== storageLease) { return { status: 'lease-lost' } } - const usageDurationMs = Math.max(0, Math.round(input.usageDurationMs)) - const usageBytes = Math.max(0, Math.round(input.usageBytes)) + const usageDurationMs = Math.round( + assertMailboxInboundNonNegativeFiniteNumber( + input.usageDurationMs, + 'usageDurationMs', + ), + ) + const usageBytes = Math.round( + assertMailboxInboundNonNegativeFiniteNumber(input.usageBytes, 'usageBytes'), + ) const next: MailboxInboundDeliverySnapshot = { ...current, state: 'received', diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts index 7346639a72..911b6b1df8 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -68,6 +68,13 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip ) .toArray()[0]?.value expect(Number(version)).toBe(mailboxSchemaVersion) + const schemaV2Indexes = [ + 'idx_email_delivery_events_reconcile_after', + 'idx_email_delivery_events_usage_effect_retry', + 'idx_email_delivery_events_subscription_effect_retry', + 'idx_email_delivery_events_stale_state', + 'idx_email_delivery_events_dedupe_provider_expires', + ] as const const indexes = state.storage.sql .exec<{ name: string }>( `SELECT name FROM sqlite_master @@ -77,23 +84,16 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip 'idx_email_delivery_events_usage_effect_retry', 'idx_email_delivery_events_subscription_effect_retry', 'idx_email_delivery_events_stale_state', - 'idx_email_delivery_events_dedupe_provider_expires', - 'idx_email_delivery_events_state_created', - 'idx_email_delivery_events_dedupe_expires' + 'idx_email_delivery_events_dedupe_provider_expires' ) ORDER BY name ASC`, ) .toArray() .map((row) => row.name) - expect(indexes).toContain('idx_email_delivery_events_reconcile_after') - expect(indexes).toContain('idx_email_delivery_events_usage_effect_retry') - expect(indexes).toContain( - 'idx_email_delivery_events_subscription_effect_retry', - ) - expect(indexes).toContain('idx_email_delivery_events_stale_state') - expect(indexes).toContain( - 'idx_email_delivery_events_dedupe_provider_expires', - ) + expect(indexes).toEqual([...schemaV2Indexes].sort()) + for (const name of schemaV2Indexes) { + expect(indexes.filter((entry) => entry === name)).toHaveLength(1) + } }, ) @@ -823,13 +823,292 @@ test('Mailbox inbound ledger warm-migrates v1 schema indexes to v2', async () => ) .toArray()[0]?.value expect(Number(version)).toBe(mailboxSchemaVersion) - const hasReconcile = state.storage.sql + const schemaV2Indexes = [ + 'idx_email_delivery_events_reconcile_after', + 'idx_email_delivery_events_usage_effect_retry', + 'idx_email_delivery_events_subscription_effect_retry', + 'idx_email_delivery_events_stale_state', + 'idx_email_delivery_events_dedupe_provider_expires', + ] as const + const indexes = state.storage.sql .exec<{ name: string }>( `SELECT name FROM sqlite_master WHERE type = 'index' - AND name = 'idx_email_delivery_events_reconcile_after'`, + AND name IN ( + 'idx_email_delivery_events_reconcile_after', + 'idx_email_delivery_events_usage_effect_retry', + 'idx_email_delivery_events_subscription_effect_retry', + 'idx_email_delivery_events_stale_state', + 'idx_email_delivery_events_dedupe_provider_expires' + ) + ORDER BY name ASC`, ) .toArray() - expect(hasReconcile.length).toBe(1) + .map((row) => row.name) + expect(indexes).toEqual([...schemaV2Indexes].sort()) + for (const name of schemaV2Indexes) { + expect(indexes.filter((entry) => entry === name)).toHaveLength(1) + } + }) +}) + +test('legacy received rows with null effect lease timestamps and null subscription state are claimable and due', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('null-lease') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-null-lease', + deliveryId: 'email-inbound-delivery:null-lease', + messageId: 'email-inbound-message:null-lease', + threadId: 'email-inbound-thread:null-lease', + }) + + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + const storage = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(storage.status).toBe('claimed') + if (storage.status !== 'claimed') throw new Error('expected claim') + const received = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: storage.delivery.storageLease!, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: 8, + now, + }) + expect(received.status).toBe('received') + if (received.status !== 'received') throw new Error('expected received') + const finalizationToken = received.delivery.finalizationToken! + expect(finalizationToken).toBeTruthy() + + // Simulate legacy/imported received row: null leases + null subscription state + // in both columns and detail_json (snapshot falls back to detail). + await runInDurableObject(stubFor(ownerId), async (_instance, state) => { + state.storage.sql.exec( + `UPDATE email_delivery_events + SET usage_effect_lease = 'legacy-usage-lease', + usage_effect_lease_at = NULL, + subscription_effect_state = NULL, + subscription_effect_lease = 'legacy-sub-lease', + subscription_effect_lease_at = NULL, + detail_json = json_remove( + json_remove( + json_remove( + json_remove( + json_remove(detail_json, '$.subscriptionEffectState'), + '$.usageEffectLeaseAt' + ), + '$.subscriptionEffectLeaseAt' + ), + '$.usageEffectLease' + ), + '$.subscriptionEffectLease' + ) + WHERE id = ? AND provider = ?`, + delivery.deliveryId, + mailboxInboundProvider, + ) + }) + + const due = await mailbox.listDueInboundEffectWork({ + ownerId, + now: '2026-07-22T00:00:01.000Z', + limit: 50, + }) + expect( + due.deliveries.some((entry) => entry.deliveryId === delivery.deliveryId), + ).toBe(true) + + const usage = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: '2026-07-22T00:00:02.000Z', + }) + expect(usage.status).toBe('claimed') + if (usage.status !== 'claimed') throw new Error('expected usage claim') + expect(usage.delivery.usageEffectLease).not.toBe('legacy-usage-lease') + + const subscription = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: '2026-07-22T00:00:03.000Z', + }) + expect(subscription.status).toBe('claimed') + if (subscription.status !== 'claimed') { + throw new Error('expected subscription claim') + } + expect(subscription.delivery.subscriptionEffectLease).not.toBe( + 'legacy-sub-lease', + ) + expect(subscription.delivery.subscriptionEffectState).toBe('processing') +}) + +test('processing subscription rows with null lease_at remain reclaimable and due', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('null-sub-lease-at') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-null-sub-lease-at', + deliveryId: 'email-inbound-delivery:null-sub-lease-at', + messageId: 'email-inbound-message:null-sub-lease-at', + threadId: 'email-inbound-thread:null-sub-lease-at', + }) + + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + const storage = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(storage.status).toBe('claimed') + if (storage.status !== 'claimed') throw new Error('expected claim') + const received = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: storage.delivery.storageLease!, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: 8, + now, + }) + expect(received.status).toBe('received') + if (received.status !== 'received') throw new Error('expected received') + const finalizationToken = received.delivery.finalizationToken! + + await runInDurableObject(stubFor(ownerId), async (_instance, state) => { + state.storage.sql.exec( + `UPDATE email_delivery_events + SET subscription_effect_state = 'processing', + subscription_effect_lease = 'stale-lease', + subscription_effect_lease_at = NULL, + detail_json = json_remove( + json_set( + json_set(detail_json, '$.subscriptionEffectState', 'processing'), + '$.subscriptionEffectLease', + 'stale-lease' + ), + '$.subscriptionEffectLeaseAt' + ) + WHERE id = ? AND provider = ?`, + delivery.deliveryId, + mailboxInboundProvider, + ) + }) + + const due = await mailbox.listDueInboundEffectWork({ + ownerId, + now: '2026-07-22T00:00:01.000Z', + limit: 50, + }) + expect( + due.deliveries.some((entry) => entry.deliveryId === delivery.deliveryId), + ).toBe(true) + + const subscription = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: '2026-07-22T00:00:02.000Z', + }) + expect(subscription.status).toBe('claimed') + if (subscription.status !== 'claimed') { + throw new Error('expected subscription claim') + } + expect(subscription.delivery.subscriptionEffectLease).not.toBe('stale-lease') +}) + +test('inbound ledger rejects non-finite expectedAttachmentCount, usageDurationMs, and usageBytes', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('non-finite') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-non-finite', + deliveryId: 'email-inbound-delivery:non-finite', + messageId: 'email-inbound-message:non-finite', + threadId: 'email-inbound-thread:non-finite', + }) + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + + await runInDurableObject(stubFor(ownerId), async (instance: Mailbox) => { + for (const value of [ + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]) { + await assertMailboxThrows( + /expectedAttachmentCount must be a finite number/, + () => + instance.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: value, + now, + }), + ) + } + }) + + const claim = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(claim.status).toBe('claimed') + if (claim.status !== 'claimed') throw new Error('expected claim') + const storageLease = claim.delivery.storageLease! + + await runInDurableObject(stubFor(ownerId), async (instance: Mailbox) => { + for (const value of [ + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]) { + await assertMailboxThrows(/usageDurationMs must be a finite number/, () => + instance.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease, + usageDurationMs: value, + usageMonth: '2026-07', + usageBytes: 8, + now, + }), + ) + await assertMailboxThrows(/usageBytes must be a finite number/, () => + instance.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: value, + now, + }), + ) + } }) }) diff --git a/packages/worker/src/email/mailbox-schema.ts b/packages/worker/src/email/mailbox-schema.ts index 7fd8ee741d..9a7690a565 100644 --- a/packages/worker/src/email/mailbox-schema.ts +++ b/packages/worker/src/email/mailbox-schema.ts @@ -331,6 +331,8 @@ export function initializeMailboxSchema(storage: DurableObjectStorage) { ON email_delivery_events(state, created_at ASC, id ASC) WHERE state IN ('pending', 'storing', 'cleaning', 'orphan-cleaned')`, ) + // Literal matches mailboxInboundDedupeProvider; keep local to avoid a + // mailbox-types ↔ ledger ↔ schema import cycle. sql.exec( `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_dedupe_provider_expires ON email_delivery_events(provider, dedupe_expires_at ASC, id ASC) From 2248adfbb86f6bd610ef48cb351e2d07b030866b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 14:31:52 +0000 Subject: [PATCH 51/62] test(email): split inbound ledger matrices Co-authored-by: Kent C. Dodds --- ...lbox-inbound-effect-ledger.workers.test.ts | 603 ++++++++++++++++++ .../mailbox-inbound-ledger-test-helpers.ts | 29 + .../mailbox-inbound-ledger.workers.test.ts | 593 +---------------- 3 files changed, 633 insertions(+), 592 deletions(-) create mode 100644 packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts create mode 100644 packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts diff --git a/packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts new file mode 100644 index 0000000000..c4f9964f41 --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts @@ -0,0 +1,603 @@ +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' +import { insertInput } from './mailbox-inbound-ledger-test-helpers.ts' +import { + mailboxInboundProvider, + type MailboxInboundDeliveryInsertInput, +} from './mailbox-inbound-ledger.ts' +import { rpcFor, stubFor, uniqueUserId } from './mailbox-test-helpers.ts' + +async function insertClaimAndReceive( + mailbox: ReturnType, + ownerId: string, + delivery: MailboxInboundDeliveryInsertInput, + now: string, +) { + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + const claim = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(claim.status).toBe('claimed') + if (claim.status !== 'claimed') throw new Error('expected claim') + const received = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: claim.delivery.storageLease!, + usageDurationMs: 25, + usageMonth: '2026-07', + usageBytes: 64, + now, + }) + expect(received.status).toBe('received') + if (received.status !== 'received') throw new Error('expected received') + return received +} + +test('Mailbox inbound usage and subscription effects enforce exactly-once leases, retry, and dead-letter', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('effect-exactly-once') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-effect-exactly-once', + deliveryId: 'email-inbound-delivery:effect-exactly-once', + messageId: 'email-inbound-message:effect-exactly-once', + threadId: 'email-inbound-thread:effect-exactly-once', + }) + const received = await insertClaimAndReceive(mailbox, ownerId, delivery, now) + const finalizationToken = received.delivery.finalizationToken! + + const usageClaim = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now, + }) + expect(usageClaim.status).toBe('claimed') + if (usageClaim.status !== 'claimed') throw new Error('expected usage claim') + const usageBusy = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + now, + }) + expect(usageBusy.status).toBe('not-claimable') + const usageDone = await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + usageEffectLease: usageClaim.delivery.usageEffectLease!, + expectedFinalizationToken: finalizationToken, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 64, + usageDurationMs: 25, + now, + }) + expect(usageDone.status).toBe('recorded') + const usageReplay = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + now, + }) + expect(usageReplay.status).toBe('already-complete') + + const subClaim = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now, + }) + expect(subClaim.status).toBe('claimed') + if (subClaim.status !== 'claimed') throw new Error('expected sub claim') + const fail1 = await mailbox.failInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: subClaim.delivery.subscriptionEffectLease!, + expectedFinalizationToken: finalizationToken, + error: 'transient-1', + now, + }) + expect(fail1.status).toBe('retry') + + const retryAt = + fail1.status === 'retry' ? fail1.delivery.subscriptionEffectRetryAt! : now + const subClaim2 = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: retryAt, + }) + expect(subClaim2.status).toBe('claimed') + if (subClaim2.status !== 'claimed') throw new Error('expected sub claim 2') + const fail2 = await mailbox.failInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: subClaim2.delivery.subscriptionEffectLease!, + expectedFinalizationToken: finalizationToken, + error: 'transient-2', + now: retryAt, + }) + expect(fail2.status).toBe('retry') + const retryAt2 = + fail2.status === 'retry' + ? fail2.delivery.subscriptionEffectRetryAt! + : retryAt + const subClaim3 = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: retryAt2, + }) + expect(subClaim3.status).toBe('claimed') + if (subClaim3.status !== 'claimed') throw new Error('expected sub claim 3') + const dead = await mailbox.failInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: subClaim3.delivery.subscriptionEffectLease!, + expectedFinalizationToken: finalizationToken, + error: 'final', + now: retryAt2, + }) + expect(dead.status).toBe('dead-letter') +}) + +test('Mailbox inbound usage and subscription effects support suppression on received deliveries', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('effect-suppress') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const suppressDelivery = insertInput(ownerId, { + fingerprint: 'fp-suppress', + deliveryId: 'email-inbound-delivery:suppress', + messageId: 'email-inbound-message:suppress', + threadId: 'email-inbound-thread:suppress', + }) + const suppressReceived = await insertClaimAndReceive( + mailbox, + ownerId, + suppressDelivery, + now, + ) + const suppressToken = suppressReceived.delivery.finalizationToken! + + const usageSuppressClaim = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: suppressDelivery.deliveryId, + now, + }) + expect(usageSuppressClaim.status).toBe('claimed') + if (usageSuppressClaim.status !== 'claimed') { + throw new Error('expected usage suppress claim') + } + expect( + ( + await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: suppressDelivery.deliveryId, + usageEffectLease: usageSuppressClaim.delivery.usageEffectLease!, + expectedFinalizationToken: suppressToken, + mode: 'suppressed', + usageMonth: '2026-07', + usageBytes: 1, + usageDurationMs: 1, + now, + }) + ).status, + ).toBe('suppressed') + const subSuppressClaim = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: suppressDelivery.deliveryId, + expectedFinalizationToken: suppressToken, + now, + }) + expect(subSuppressClaim.status).toBe('claimed') + if (subSuppressClaim.status !== 'claimed') { + throw new Error('expected sub suppress claim') + } + expect( + ( + await mailbox.completeInboundSubscriptionEffect({ + ownerId, + deliveryId: suppressDelivery.deliveryId, + subscriptionEffectLease: + subSuppressClaim.delivery.subscriptionEffectLease!, + expectedFinalizationToken: suppressToken, + mode: 'suppressed', + suppressionReason: 'quarantine', + now, + }) + ).status, + ).toBe('suppressed') +}) + +test('stale effect workers cannot complete or fail after storage reclaim re-finalization', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('reclaim-fence') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-reclaim-fence', + deliveryId: 'email-inbound-delivery:reclaim-fence', + messageId: 'email-inbound-message:reclaim-fence', + threadId: 'email-inbound-thread:reclaim-fence', + }) + + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + const firstClaim = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(firstClaim.status).toBe('claimed') + if (firstClaim.status !== 'claimed') throw new Error('expected first claim') + const firstReceived = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: firstClaim.delivery.storageLease!, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: 8, + now, + }) + expect(firstReceived.status).toBe('received') + if (firstReceived.status !== 'received') throw new Error('expected received') + const staleToken = firstReceived.delivery.finalizationToken! + expect(staleToken).toBeTruthy() + + const staleUsageClaim = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: staleToken, + now, + }) + expect(staleUsageClaim.status).toBe('claimed') + if (staleUsageClaim.status !== 'claimed') { + throw new Error('expected stale usage claim') + } + const staleUsageLease = staleUsageClaim.delivery.usageEffectLease! + const staleSubClaim = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: staleToken, + now, + }) + expect(staleSubClaim.status).toBe('claimed') + if (staleSubClaim.status !== 'claimed') { + throw new Error('expected stale subscription claim') + } + const staleSubLease = staleSubClaim.delivery.subscriptionEffectLease! + + // Reclaim storage (no message row) — clears finalization + effect leases. + const reclaim = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now: '2026-07-22T00:01:00.000Z', + }) + expect(reclaim.status).toBe('claimed') + if (reclaim.status !== 'claimed') throw new Error('expected reclaim') + expect(reclaim.delivery.finalizationToken).toBeUndefined() + expect(reclaim.delivery.usageEffectLease).toBeUndefined() + expect(reclaim.delivery.subscriptionEffectLease).toBeUndefined() + expect(reclaim.delivery.subscriptionEffectState).toBe('pending') + + const secondReceived = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: reclaim.delivery.storageLease!, + usageDurationMs: 9, + usageMonth: '2026-07', + usageBytes: 16, + now: '2026-07-22T00:01:01.000Z', + }) + expect(secondReceived.status).toBe('received') + if (secondReceived.status !== 'received') { + throw new Error('expected second received') + } + const freshToken = secondReceived.delivery.finalizationToken! + expect(freshToken).toBeTruthy() + expect(freshToken).not.toBe(staleToken) + + // Stale workers with prior token/leases cannot complete or fail. + expect( + ( + await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + usageEffectLease: staleUsageLease, + expectedFinalizationToken: staleToken, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 8, + usageDurationMs: 5, + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + expect( + ( + await mailbox.completeInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: staleSubLease, + expectedFinalizationToken: staleToken, + mode: 'complete', + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + expect( + ( + await mailbox.failInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: staleSubLease, + expectedFinalizationToken: staleToken, + error: 'stale-fail', + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + // Stale lease + fresh token still loses (lease cleared on reclaim). + expect( + ( + await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + usageEffectLease: staleUsageLease, + expectedFinalizationToken: freshToken, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 8, + usageDurationMs: 5, + now: '2026-07-22T00:01:02.000Z', + }) + ).status, + ).toBe('lease-lost') + + // Fresh worker with the new finalization token can claim and complete. + const freshUsageClaim = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: freshToken, + now: '2026-07-22T00:01:03.000Z', + }) + expect(freshUsageClaim.status).toBe('claimed') + if (freshUsageClaim.status !== 'claimed') { + throw new Error('expected fresh usage claim') + } + expect( + ( + await mailbox.completeInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + usageEffectLease: freshUsageClaim.delivery.usageEffectLease!, + expectedFinalizationToken: freshToken, + mode: 'recorded', + usageMonth: '2026-07', + usageBytes: 16, + usageDurationMs: 9, + now: '2026-07-22T00:01:03.000Z', + }) + ).status, + ).toBe('recorded') + + const freshSubClaim = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: freshToken, + now: '2026-07-22T00:01:04.000Z', + }) + expect(freshSubClaim.status).toBe('claimed') + if (freshSubClaim.status !== 'claimed') { + throw new Error('expected fresh subscription claim') + } + expect( + ( + await mailbox.completeInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + subscriptionEffectLease: + freshSubClaim.delivery.subscriptionEffectLease!, + expectedFinalizationToken: freshToken, + mode: 'complete', + now: '2026-07-22T00:01:04.000Z', + }) + ).status, + ).toBe('complete') +}) + +test('legacy received rows with null effect lease timestamps and null subscription state are claimable and due', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('null-lease') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-null-lease', + deliveryId: 'email-inbound-delivery:null-lease', + messageId: 'email-inbound-message:null-lease', + threadId: 'email-inbound-thread:null-lease', + }) + + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + const storage = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(storage.status).toBe('claimed') + if (storage.status !== 'claimed') throw new Error('expected claim') + const received = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: storage.delivery.storageLease!, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: 8, + now, + }) + expect(received.status).toBe('received') + if (received.status !== 'received') throw new Error('expected received') + const finalizationToken = received.delivery.finalizationToken! + expect(finalizationToken).toBeTruthy() + + // Simulate legacy/imported received row: null leases + null subscription state + // in both columns and detail_json (snapshot falls back to detail). + await runInDurableObject(stubFor(ownerId), async (_instance, state) => { + state.storage.sql.exec( + `UPDATE email_delivery_events + SET usage_effect_lease = 'legacy-usage-lease', + usage_effect_lease_at = NULL, + subscription_effect_state = NULL, + subscription_effect_lease = 'legacy-sub-lease', + subscription_effect_lease_at = NULL, + detail_json = json_remove( + json_remove( + json_remove( + json_remove( + json_remove(detail_json, '$.subscriptionEffectState'), + '$.usageEffectLeaseAt' + ), + '$.subscriptionEffectLeaseAt' + ), + '$.usageEffectLease' + ), + '$.subscriptionEffectLease' + ) + WHERE id = ? AND provider = ?`, + delivery.deliveryId, + mailboxInboundProvider, + ) + }) + + const due = await mailbox.listDueInboundEffectWork({ + ownerId, + now: '2026-07-22T00:00:01.000Z', + limit: 50, + }) + expect( + due.deliveries.some((entry) => entry.deliveryId === delivery.deliveryId), + ).toBe(true) + + const usage = await mailbox.claimInboundUsageEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: '2026-07-22T00:00:02.000Z', + }) + expect(usage.status).toBe('claimed') + if (usage.status !== 'claimed') throw new Error('expected usage claim') + expect(usage.delivery.usageEffectLease).not.toBe('legacy-usage-lease') + + const subscription = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: '2026-07-22T00:00:03.000Z', + }) + expect(subscription.status).toBe('claimed') + if (subscription.status !== 'claimed') { + throw new Error('expected subscription claim') + } + expect(subscription.delivery.subscriptionEffectLease).not.toBe( + 'legacy-sub-lease', + ) + expect(subscription.delivery.subscriptionEffectState).toBe('processing') +}) + +test('processing subscription rows with null lease_at remain reclaimable and due', async () => { + silenceIncidentalRuntimeWarnings() + const ownerId = uniqueUserId('null-sub-lease-at') + const mailbox = rpcFor(ownerId) + const now = '2026-07-22T00:00:00.000Z' + const delivery = insertInput(ownerId, { + fingerprint: 'fp-null-sub-lease-at', + deliveryId: 'email-inbound-delivery:null-sub-lease-at', + messageId: 'email-inbound-message:null-sub-lease-at', + threadId: 'email-inbound-thread:null-sub-lease-at', + }) + + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery, + now, + }) + const storage = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now, + }) + expect(storage.status).toBe('claimed') + if (storage.status !== 'claimed') throw new Error('expected claim') + const received = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: storage.delivery.storageLease!, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: 8, + now, + }) + expect(received.status).toBe('received') + if (received.status !== 'received') throw new Error('expected received') + const finalizationToken = received.delivery.finalizationToken! + + await runInDurableObject(stubFor(ownerId), async (_instance, state) => { + state.storage.sql.exec( + `UPDATE email_delivery_events + SET subscription_effect_state = 'processing', + subscription_effect_lease = 'stale-lease', + subscription_effect_lease_at = NULL, + detail_json = json_remove( + json_set( + json_set(detail_json, '$.subscriptionEffectState', 'processing'), + '$.subscriptionEffectLease', + 'stale-lease' + ), + '$.subscriptionEffectLeaseAt' + ) + WHERE id = ? AND provider = ?`, + delivery.deliveryId, + mailboxInboundProvider, + ) + }) + + const due = await mailbox.listDueInboundEffectWork({ + ownerId, + now: '2026-07-22T00:00:01.000Z', + limit: 50, + }) + expect( + due.deliveries.some((entry) => entry.deliveryId === delivery.deliveryId), + ).toBe(true) + + const subscription = await mailbox.claimInboundSubscriptionEffect({ + ownerId, + deliveryId: delivery.deliveryId, + expectedFinalizationToken: finalizationToken, + now: '2026-07-22T00:00:02.000Z', + }) + expect(subscription.status).toBe('claimed') + if (subscription.status !== 'claimed') { + throw new Error('expected subscription claim') + } + expect(subscription.delivery.subscriptionEffectLease).not.toBe('stale-lease') +}) diff --git a/packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts b/packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts new file mode 100644 index 0000000000..fe43e77969 --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts @@ -0,0 +1,29 @@ +import { emailRawMimeKey } from './blob-keys.ts' +import { type MailboxInboundDeliveryInsertInput } from './mailbox-inbound-ledger.ts' + +export function insertInput( + ownerId: string, + overrides?: Partial, +): MailboxInboundDeliveryInsertInput { + const deliveryId = + overrides?.deliveryId ?? `email-inbound-delivery:${crypto.randomUUID()}` + const messageId = + overrides?.messageId ?? `email-inbound-message:${crypto.randomUUID()}` + const fingerprint = + overrides?.fingerprint ?? crypto.randomUUID().replace(/-/g, '') + return { + fingerprint, + deliveryId, + messageId, + threadId: + overrides?.threadId ?? `email-inbound-thread:${crypto.randomUUID()}`, + rawMimeKey: emailRawMimeKey(ownerId, messageId), + inboxId: overrides?.inboxId ?? `inbox-${crypto.randomUUID()}`, + recipient: overrides?.recipient ?? 'owner@example.com', + envelopeFrom: overrides?.envelopeFrom ?? 'sender@example.com', + quotaDay: overrides?.quotaDay ?? '2026-07-22', + dedupeExpiresAt: overrides?.dedupeExpiresAt ?? '2026-07-24T00:00:00.000Z', + usageStartedAt: overrides?.usageStartedAt, + provider: overrides?.provider, + } +} diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts index 911b6b1df8..b1038a2701 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -3,11 +3,11 @@ import { expect, test } from 'vitest' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' import { emailRawMimeKey } from './blob-keys.ts' import { Mailbox } from './mailbox-do.ts' +import { insertInput } from './mailbox-inbound-ledger-test-helpers.ts' import { mailboxInboundDedupeProvider, mailboxInboundProvider, mailboxInboundStorageLeaseMs, - type MailboxInboundDeliveryInsertInput, } from './mailbox-inbound-ledger.ts' import { initializeMailboxSchema } from './mailbox-schema.ts' import { @@ -21,33 +21,6 @@ import { uniqueUserId, } from './mailbox-test-helpers.ts' -function insertInput( - ownerId: string, - overrides?: Partial, -): MailboxInboundDeliveryInsertInput { - const deliveryId = - overrides?.deliveryId ?? `email-inbound-delivery:${crypto.randomUUID()}` - const messageId = - overrides?.messageId ?? `email-inbound-message:${crypto.randomUUID()}` - const fingerprint = - overrides?.fingerprint ?? crypto.randomUUID().replace(/-/g, '') - return { - fingerprint, - deliveryId, - messageId, - threadId: - overrides?.threadId ?? `email-inbound-thread:${crypto.randomUUID()}`, - rawMimeKey: emailRawMimeKey(ownerId, messageId), - inboxId: overrides?.inboxId ?? `inbox-${crypto.randomUUID()}`, - recipient: overrides?.recipient ?? 'owner@example.com', - envelopeFrom: overrides?.envelopeFrom ?? 'sender@example.com', - quotaDay: overrides?.quotaDay ?? '2026-07-22', - dedupeExpiresAt: overrides?.dedupeExpiresAt ?? '2026-07-24T00:00:00.000Z', - usageStartedAt: overrides?.usageStartedAt, - provider: overrides?.provider, - } -} - test('Mailbox inbound ledger CAS covers USER transition matrix without live flip', async () => { silenceIncidentalRuntimeWarnings() const ownerA = uniqueUserId('ledger-a') @@ -259,185 +232,6 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip }) expect(rejected.status).toBe('rejected') - // Usage + subscription effect exactly-once leases / retry / dead-letter. - const usageClaim = await mailboxA.claimInboundUsageEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: received.delivery.finalizationToken, - now: takeoverNow, - }) - expect(usageClaim.status).toBe('claimed') - if (usageClaim.status !== 'claimed') throw new Error('expected usage claim') - const usageBusy = await mailboxA.claimInboundUsageEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - now: takeoverNow, - }) - expect(usageBusy.status).toBe('not-claimable') - const finalizationToken = received.delivery.finalizationToken! - const usageDone = await mailboxA.completeInboundUsageEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - usageEffectLease: usageClaim.delivery.usageEffectLease!, - expectedFinalizationToken: finalizationToken, - mode: 'recorded', - usageMonth: '2026-07', - usageBytes: 64, - usageDurationMs: 25, - now: takeoverNow, - }) - expect(usageDone.status).toBe('recorded') - const usageReplay = await mailboxA.claimInboundUsageEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - now: takeoverNow, - }) - expect(usageReplay.status).toBe('already-complete') - - const subClaim = await mailboxA.claimInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: finalizationToken, - now: takeoverNow, - }) - expect(subClaim.status).toBe('claimed') - if (subClaim.status !== 'claimed') throw new Error('expected sub claim') - const fail1 = await mailboxA.failInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - subscriptionEffectLease: subClaim.delivery.subscriptionEffectLease!, - expectedFinalizationToken: finalizationToken, - error: 'transient-1', - now: takeoverNow, - }) - expect(fail1.status).toBe('retry') - - const retryAt = - fail1.status === 'retry' - ? fail1.delivery.subscriptionEffectRetryAt! - : takeoverNow - const subClaim2 = await mailboxA.claimInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: finalizationToken, - now: retryAt, - }) - expect(subClaim2.status).toBe('claimed') - if (subClaim2.status !== 'claimed') throw new Error('expected sub claim 2') - const fail2 = await mailboxA.failInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - subscriptionEffectLease: subClaim2.delivery.subscriptionEffectLease!, - expectedFinalizationToken: finalizationToken, - error: 'transient-2', - now: retryAt, - }) - expect(fail2.status).toBe('retry') - const retryAt2 = - fail2.status === 'retry' - ? fail2.delivery.subscriptionEffectRetryAt! - : retryAt - const subClaim3 = await mailboxA.claimInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: finalizationToken, - now: retryAt2, - }) - expect(subClaim3.status).toBe('claimed') - if (subClaim3.status !== 'claimed') throw new Error('expected sub claim 3') - const dead = await mailboxA.failInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: delivery.deliveryId, - subscriptionEffectLease: subClaim3.delivery.subscriptionEffectLease!, - expectedFinalizationToken: finalizationToken, - error: 'final', - now: retryAt2, - }) - expect(dead.status).toBe('dead-letter') - - // Suppression path on a fresh received delivery. - const suppressDelivery = insertInput(ownerA, { - fingerprint: 'fp-suppress', - deliveryId: 'email-inbound-delivery:suppress', - messageId: 'email-inbound-message:suppress', - threadId: 'email-inbound-thread:suppress', - }) - await mailboxA.insertChargedPendingInboundDelivery({ - ownerId: ownerA, - delivery: suppressDelivery, - now, - }) - const suppressClaim = await mailboxA.claimInboundDeliveryStorage({ - ownerId: ownerA, - deliveryId: suppressDelivery.deliveryId, - expectedAttachmentCount: 0, - now, - }) - expect(suppressClaim.status).toBe('claimed') - if (suppressClaim.status !== 'claimed') throw new Error('expected claim') - const suppressReceived = await mailboxA.markInboundDeliveryReceived({ - ownerId: ownerA, - deliveryId: suppressDelivery.deliveryId, - storageLease: suppressClaim.delivery.storageLease!, - usageDurationMs: 1, - usageMonth: '2026-07', - usageBytes: 1, - now, - }) - expect(suppressReceived.status).toBe('received') - const usageSuppressClaim = await mailboxA.claimInboundUsageEffect({ - ownerId: ownerA, - deliveryId: suppressDelivery.deliveryId, - now, - }) - expect(usageSuppressClaim.status).toBe('claimed') - if (usageSuppressClaim.status !== 'claimed') { - throw new Error('expected usage suppress claim') - } - const suppressToken = - suppressReceived.status === 'received' - ? suppressReceived.delivery.finalizationToken! - : '' - expect( - ( - await mailboxA.completeInboundUsageEffect({ - ownerId: ownerA, - deliveryId: suppressDelivery.deliveryId, - usageEffectLease: usageSuppressClaim.delivery.usageEffectLease!, - expectedFinalizationToken: suppressToken, - mode: 'suppressed', - usageMonth: '2026-07', - usageBytes: 1, - usageDurationMs: 1, - now, - }) - ).status, - ).toBe('suppressed') - const subSuppressClaim = await mailboxA.claimInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: suppressDelivery.deliveryId, - expectedFinalizationToken: suppressToken, - now, - }) - expect(subSuppressClaim.status).toBe('claimed') - if (subSuppressClaim.status !== 'claimed') { - throw new Error('expected sub suppress claim') - } - expect( - ( - await mailboxA.completeInboundSubscriptionEffect({ - ownerId: ownerA, - deliveryId: suppressDelivery.deliveryId, - subscriptionEffectLease: - subSuppressClaim.delivery.subscriptionEffectLease!, - expectedFinalizationToken: suppressToken, - mode: 'suppressed', - suppressionReason: 'quarantine', - now, - }) - ).status, - ).toBe('suppressed') - // Due listing + owner isolation. const stalePending = insertInput(ownerA, { fingerprint: 'fp-stale', @@ -586,208 +380,6 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip }) }) -test('stale effect workers cannot complete or fail after storage reclaim re-finalization', async () => { - silenceIncidentalRuntimeWarnings() - const ownerId = uniqueUserId('reclaim-fence') - const mailbox = rpcFor(ownerId) - const now = '2026-07-22T00:00:00.000Z' - const delivery = insertInput(ownerId, { - fingerprint: 'fp-reclaim-fence', - deliveryId: 'email-inbound-delivery:reclaim-fence', - messageId: 'email-inbound-message:reclaim-fence', - threadId: 'email-inbound-thread:reclaim-fence', - }) - - await mailbox.insertChargedPendingInboundDelivery({ - ownerId, - delivery, - now, - }) - const firstClaim = await mailbox.claimInboundDeliveryStorage({ - ownerId, - deliveryId: delivery.deliveryId, - expectedAttachmentCount: 0, - now, - }) - expect(firstClaim.status).toBe('claimed') - if (firstClaim.status !== 'claimed') throw new Error('expected first claim') - const firstReceived = await mailbox.markInboundDeliveryReceived({ - ownerId, - deliveryId: delivery.deliveryId, - storageLease: firstClaim.delivery.storageLease!, - usageDurationMs: 5, - usageMonth: '2026-07', - usageBytes: 8, - now, - }) - expect(firstReceived.status).toBe('received') - if (firstReceived.status !== 'received') throw new Error('expected received') - const staleToken = firstReceived.delivery.finalizationToken! - expect(staleToken).toBeTruthy() - - const staleUsageClaim = await mailbox.claimInboundUsageEffect({ - ownerId, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: staleToken, - now, - }) - expect(staleUsageClaim.status).toBe('claimed') - if (staleUsageClaim.status !== 'claimed') { - throw new Error('expected stale usage claim') - } - const staleUsageLease = staleUsageClaim.delivery.usageEffectLease! - const staleSubClaim = await mailbox.claimInboundSubscriptionEffect({ - ownerId, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: staleToken, - now, - }) - expect(staleSubClaim.status).toBe('claimed') - if (staleSubClaim.status !== 'claimed') { - throw new Error('expected stale subscription claim') - } - const staleSubLease = staleSubClaim.delivery.subscriptionEffectLease! - - // Reclaim storage (no message row) — clears finalization + effect leases. - const reclaim = await mailbox.claimInboundDeliveryStorage({ - ownerId, - deliveryId: delivery.deliveryId, - expectedAttachmentCount: 0, - now: '2026-07-22T00:01:00.000Z', - }) - expect(reclaim.status).toBe('claimed') - if (reclaim.status !== 'claimed') throw new Error('expected reclaim') - expect(reclaim.delivery.finalizationToken).toBeUndefined() - expect(reclaim.delivery.usageEffectLease).toBeUndefined() - expect(reclaim.delivery.subscriptionEffectLease).toBeUndefined() - expect(reclaim.delivery.subscriptionEffectState).toBe('pending') - - const secondReceived = await mailbox.markInboundDeliveryReceived({ - ownerId, - deliveryId: delivery.deliveryId, - storageLease: reclaim.delivery.storageLease!, - usageDurationMs: 9, - usageMonth: '2026-07', - usageBytes: 16, - now: '2026-07-22T00:01:01.000Z', - }) - expect(secondReceived.status).toBe('received') - if (secondReceived.status !== 'received') { - throw new Error('expected second received') - } - const freshToken = secondReceived.delivery.finalizationToken! - expect(freshToken).toBeTruthy() - expect(freshToken).not.toBe(staleToken) - - // Stale workers with prior token/leases cannot complete or fail. - expect( - ( - await mailbox.completeInboundUsageEffect({ - ownerId, - deliveryId: delivery.deliveryId, - usageEffectLease: staleUsageLease, - expectedFinalizationToken: staleToken, - mode: 'recorded', - usageMonth: '2026-07', - usageBytes: 8, - usageDurationMs: 5, - now: '2026-07-22T00:01:02.000Z', - }) - ).status, - ).toBe('lease-lost') - expect( - ( - await mailbox.completeInboundSubscriptionEffect({ - ownerId, - deliveryId: delivery.deliveryId, - subscriptionEffectLease: staleSubLease, - expectedFinalizationToken: staleToken, - mode: 'complete', - now: '2026-07-22T00:01:02.000Z', - }) - ).status, - ).toBe('lease-lost') - expect( - ( - await mailbox.failInboundSubscriptionEffect({ - ownerId, - deliveryId: delivery.deliveryId, - subscriptionEffectLease: staleSubLease, - expectedFinalizationToken: staleToken, - error: 'stale-fail', - now: '2026-07-22T00:01:02.000Z', - }) - ).status, - ).toBe('lease-lost') - // Stale lease + fresh token still loses (lease cleared on reclaim). - expect( - ( - await mailbox.completeInboundUsageEffect({ - ownerId, - deliveryId: delivery.deliveryId, - usageEffectLease: staleUsageLease, - expectedFinalizationToken: freshToken, - mode: 'recorded', - usageMonth: '2026-07', - usageBytes: 8, - usageDurationMs: 5, - now: '2026-07-22T00:01:02.000Z', - }) - ).status, - ).toBe('lease-lost') - - // Fresh worker with the new finalization token can claim and complete. - const freshUsageClaim = await mailbox.claimInboundUsageEffect({ - ownerId, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: freshToken, - now: '2026-07-22T00:01:03.000Z', - }) - expect(freshUsageClaim.status).toBe('claimed') - if (freshUsageClaim.status !== 'claimed') { - throw new Error('expected fresh usage claim') - } - expect( - ( - await mailbox.completeInboundUsageEffect({ - ownerId, - deliveryId: delivery.deliveryId, - usageEffectLease: freshUsageClaim.delivery.usageEffectLease!, - expectedFinalizationToken: freshToken, - mode: 'recorded', - usageMonth: '2026-07', - usageBytes: 16, - usageDurationMs: 9, - now: '2026-07-22T00:01:03.000Z', - }) - ).status, - ).toBe('recorded') - - const freshSubClaim = await mailbox.claimInboundSubscriptionEffect({ - ownerId, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: freshToken, - now: '2026-07-22T00:01:04.000Z', - }) - expect(freshSubClaim.status).toBe('claimed') - if (freshSubClaim.status !== 'claimed') { - throw new Error('expected fresh subscription claim') - } - expect( - ( - await mailbox.completeInboundSubscriptionEffect({ - ownerId, - deliveryId: delivery.deliveryId, - subscriptionEffectLease: - freshSubClaim.delivery.subscriptionEffectLease!, - expectedFinalizationToken: freshToken, - mode: 'complete', - now: '2026-07-22T00:01:04.000Z', - }) - ).status, - ).toBe('complete') -}) - test('Mailbox inbound ledger warm-migrates v1 schema indexes to v2', async () => { silenceIncidentalRuntimeWarnings() const ownerId = uniqueUserId('warm-v2') @@ -852,189 +444,6 @@ test('Mailbox inbound ledger warm-migrates v1 schema indexes to v2', async () => }) }) -test('legacy received rows with null effect lease timestamps and null subscription state are claimable and due', async () => { - silenceIncidentalRuntimeWarnings() - const ownerId = uniqueUserId('null-lease') - const mailbox = rpcFor(ownerId) - const now = '2026-07-22T00:00:00.000Z' - const delivery = insertInput(ownerId, { - fingerprint: 'fp-null-lease', - deliveryId: 'email-inbound-delivery:null-lease', - messageId: 'email-inbound-message:null-lease', - threadId: 'email-inbound-thread:null-lease', - }) - - await mailbox.insertChargedPendingInboundDelivery({ - ownerId, - delivery, - now, - }) - const storage = await mailbox.claimInboundDeliveryStorage({ - ownerId, - deliveryId: delivery.deliveryId, - expectedAttachmentCount: 0, - now, - }) - expect(storage.status).toBe('claimed') - if (storage.status !== 'claimed') throw new Error('expected claim') - const received = await mailbox.markInboundDeliveryReceived({ - ownerId, - deliveryId: delivery.deliveryId, - storageLease: storage.delivery.storageLease!, - usageDurationMs: 5, - usageMonth: '2026-07', - usageBytes: 8, - now, - }) - expect(received.status).toBe('received') - if (received.status !== 'received') throw new Error('expected received') - const finalizationToken = received.delivery.finalizationToken! - expect(finalizationToken).toBeTruthy() - - // Simulate legacy/imported received row: null leases + null subscription state - // in both columns and detail_json (snapshot falls back to detail). - await runInDurableObject(stubFor(ownerId), async (_instance, state) => { - state.storage.sql.exec( - `UPDATE email_delivery_events - SET usage_effect_lease = 'legacy-usage-lease', - usage_effect_lease_at = NULL, - subscription_effect_state = NULL, - subscription_effect_lease = 'legacy-sub-lease', - subscription_effect_lease_at = NULL, - detail_json = json_remove( - json_remove( - json_remove( - json_remove( - json_remove(detail_json, '$.subscriptionEffectState'), - '$.usageEffectLeaseAt' - ), - '$.subscriptionEffectLeaseAt' - ), - '$.usageEffectLease' - ), - '$.subscriptionEffectLease' - ) - WHERE id = ? AND provider = ?`, - delivery.deliveryId, - mailboxInboundProvider, - ) - }) - - const due = await mailbox.listDueInboundEffectWork({ - ownerId, - now: '2026-07-22T00:00:01.000Z', - limit: 50, - }) - expect( - due.deliveries.some((entry) => entry.deliveryId === delivery.deliveryId), - ).toBe(true) - - const usage = await mailbox.claimInboundUsageEffect({ - ownerId, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: finalizationToken, - now: '2026-07-22T00:00:02.000Z', - }) - expect(usage.status).toBe('claimed') - if (usage.status !== 'claimed') throw new Error('expected usage claim') - expect(usage.delivery.usageEffectLease).not.toBe('legacy-usage-lease') - - const subscription = await mailbox.claimInboundSubscriptionEffect({ - ownerId, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: finalizationToken, - now: '2026-07-22T00:00:03.000Z', - }) - expect(subscription.status).toBe('claimed') - if (subscription.status !== 'claimed') { - throw new Error('expected subscription claim') - } - expect(subscription.delivery.subscriptionEffectLease).not.toBe( - 'legacy-sub-lease', - ) - expect(subscription.delivery.subscriptionEffectState).toBe('processing') -}) - -test('processing subscription rows with null lease_at remain reclaimable and due', async () => { - silenceIncidentalRuntimeWarnings() - const ownerId = uniqueUserId('null-sub-lease-at') - const mailbox = rpcFor(ownerId) - const now = '2026-07-22T00:00:00.000Z' - const delivery = insertInput(ownerId, { - fingerprint: 'fp-null-sub-lease-at', - deliveryId: 'email-inbound-delivery:null-sub-lease-at', - messageId: 'email-inbound-message:null-sub-lease-at', - threadId: 'email-inbound-thread:null-sub-lease-at', - }) - - await mailbox.insertChargedPendingInboundDelivery({ - ownerId, - delivery, - now, - }) - const storage = await mailbox.claimInboundDeliveryStorage({ - ownerId, - deliveryId: delivery.deliveryId, - expectedAttachmentCount: 0, - now, - }) - expect(storage.status).toBe('claimed') - if (storage.status !== 'claimed') throw new Error('expected claim') - const received = await mailbox.markInboundDeliveryReceived({ - ownerId, - deliveryId: delivery.deliveryId, - storageLease: storage.delivery.storageLease!, - usageDurationMs: 5, - usageMonth: '2026-07', - usageBytes: 8, - now, - }) - expect(received.status).toBe('received') - if (received.status !== 'received') throw new Error('expected received') - const finalizationToken = received.delivery.finalizationToken! - - await runInDurableObject(stubFor(ownerId), async (_instance, state) => { - state.storage.sql.exec( - `UPDATE email_delivery_events - SET subscription_effect_state = 'processing', - subscription_effect_lease = 'stale-lease', - subscription_effect_lease_at = NULL, - detail_json = json_remove( - json_set( - json_set(detail_json, '$.subscriptionEffectState', 'processing'), - '$.subscriptionEffectLease', - 'stale-lease' - ), - '$.subscriptionEffectLeaseAt' - ) - WHERE id = ? AND provider = ?`, - delivery.deliveryId, - mailboxInboundProvider, - ) - }) - - const due = await mailbox.listDueInboundEffectWork({ - ownerId, - now: '2026-07-22T00:00:01.000Z', - limit: 50, - }) - expect( - due.deliveries.some((entry) => entry.deliveryId === delivery.deliveryId), - ).toBe(true) - - const subscription = await mailbox.claimInboundSubscriptionEffect({ - ownerId, - deliveryId: delivery.deliveryId, - expectedFinalizationToken: finalizationToken, - now: '2026-07-22T00:00:02.000Z', - }) - expect(subscription.status).toBe('claimed') - if (subscription.status !== 'claimed') { - throw new Error('expected subscription claim') - } - expect(subscription.delivery.subscriptionEffectLease).not.toBe('stale-lease') -}) - test('inbound ledger rejects non-finite expectedAttachmentCount, usageDurationMs, and usageBytes', async () => { silenceIncidentalRuntimeWarnings() const ownerId = uniqueUserId('non-finite') From 36c137080504685eaef8d3fa7953eaafa5d780cf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 14:41:02 +0000 Subject: [PATCH 52/62] fix(email): reject negative inbound usage Co-authored-by: Kent C. Dodds --- .../email/mailbox-inbound-ledger-shared.ts | 8 ++-- .../mailbox-inbound-ledger-test-helpers.ts | 2 +- .../email/mailbox-inbound-ledger.node.test.ts | 5 +- .../mailbox-inbound-ledger.workers.test.ts | 46 ++++++++++--------- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/packages/worker/src/email/mailbox-inbound-ledger-shared.ts b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts index 2ce1d0cf2a..3a87b0de59 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger-shared.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts @@ -374,13 +374,13 @@ export function normalizeMailboxInboundLimit(limit: number | undefined) { return Math.min(Math.max(Math.trunc(limit), 1), 100) } -/** Non-negative finite number; rejects NaN/Infinity before floor/round. */ +/** Non-negative finite number; rejects invalid values before floor/round. */ export function assertMailboxInboundNonNegativeFiniteNumber( value: unknown, label: string, ): number { - if (typeof value !== 'number' || !Number.isFinite(value)) { - throw new Error(`Mailbox ${label} must be a finite number.`) + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`Mailbox ${label} must be a non-negative finite number.`) } - return Math.max(0, value) + return value } diff --git a/packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts b/packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts index fe43e77969..3b7daaae78 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger-test-helpers.ts @@ -17,7 +17,7 @@ export function insertInput( messageId, threadId: overrides?.threadId ?? `email-inbound-thread:${crypto.randomUUID()}`, - rawMimeKey: emailRawMimeKey(ownerId, messageId), + rawMimeKey: overrides?.rawMimeKey ?? emailRawMimeKey(ownerId, messageId), inboxId: overrides?.inboxId ?? `inbox-${crypto.randomUUID()}`, recipient: overrides?.recipient ?? 'owner@example.com', envelopeFrom: overrides?.envelopeFrom ?? 'sender@example.com', diff --git a/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts index 45e8bd0cfa..4e05deb4a5 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.node.test.ts @@ -61,11 +61,12 @@ test('storage lease takeover predicate matches D1 stale window', () => { expect(staleLeaseAt < expiredBefore).toBe(true) }) -test('assertMailboxInboundNonNegativeFiniteNumber rejects NaN and Infinity', () => { +test('assertMailboxInboundNonNegativeFiniteNumber rejects invalid values', () => { for (const value of [ Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, + -1, ]) { for (const label of [ 'expectedAttachmentCount', @@ -74,7 +75,7 @@ test('assertMailboxInboundNonNegativeFiniteNumber rejects NaN and Infinity', () ] as const) { expect(() => assertMailboxInboundNonNegativeFiniteNumber(value, label), - ).toThrow(new RegExp(`${label} must be a finite number`)) + ).toThrow(new RegExp(`${label} must be a non-negative finite number`)) } } expect(assertMailboxInboundNonNegativeFiniteNumber(0, 'usageBytes')).toBe(0) diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts index b1038a2701..ebc255abcd 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -468,7 +468,7 @@ test('inbound ledger rejects non-finite expectedAttachmentCount, usageDurationMs Number.NEGATIVE_INFINITY, ]) { await assertMailboxThrows( - /expectedAttachmentCount must be a finite number/, + /expectedAttachmentCount must be a non-negative finite number/, () => instance.claimInboundDeliveryStorage({ ownerId, @@ -496,27 +496,31 @@ test('inbound ledger rejects non-finite expectedAttachmentCount, usageDurationMs Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, ]) { - await assertMailboxThrows(/usageDurationMs must be a finite number/, () => - instance.markInboundDeliveryReceived({ - ownerId, - deliveryId: delivery.deliveryId, - storageLease, - usageDurationMs: value, - usageMonth: '2026-07', - usageBytes: 8, - now, - }), + await assertMailboxThrows( + /usageDurationMs must be a non-negative finite number/, + () => + instance.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease, + usageDurationMs: value, + usageMonth: '2026-07', + usageBytes: 8, + now, + }), ) - await assertMailboxThrows(/usageBytes must be a finite number/, () => - instance.markInboundDeliveryReceived({ - ownerId, - deliveryId: delivery.deliveryId, - storageLease, - usageDurationMs: 5, - usageMonth: '2026-07', - usageBytes: value, - now, - }), + await assertMailboxThrows( + /usageBytes must be a non-negative finite number/, + () => + instance.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease, + usageDurationMs: 5, + usageMonth: '2026-07', + usageBytes: value, + now, + }), ) } }) From a6fd1ad2e4a47054d5723167f2f8aaf19c4b1c60 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 14:49:11 +0000 Subject: [PATCH 53/62] fix(email): reset effects on inbound refinalization Co-authored-by: Kent C. Dodds --- ...lbox-inbound-effect-ledger.workers.test.ts | 25 +++++++++++++++++++ .../src/email/mailbox-inbound-ledger.ts | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts index c4f9964f41..cb578c6648 100644 --- a/packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-effect-ledger.workers.test.ts @@ -417,6 +417,31 @@ test('stale effect workers cannot complete or fail after storage reclaim re-fina }) ).status, ).toBe('complete') + + const terminalReclaim = await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount: 0, + now: '2026-07-22T00:02:00.000Z', + }) + expect(terminalReclaim.status).toBe('claimed') + if (terminalReclaim.status !== 'claimed') { + throw new Error('expected terminal-state reclaim') + } + const thirdReceived = await mailbox.markInboundDeliveryReceived({ + ownerId, + deliveryId: delivery.deliveryId, + storageLease: terminalReclaim.delivery.storageLease!, + usageDurationMs: 9, + usageMonth: '2026-07', + usageBytes: 16, + now: '2026-07-22T00:02:01.000Z', + }) + expect(thirdReceived.status).toBe('received') + if (thirdReceived.status !== 'received') { + throw new Error('expected third received') + } + expect(thirdReceived.delivery.subscriptionEffectState).toBe('pending') }) test('legacy received rows with null effect lease timestamps and null subscription state are claimable and due', async () => { diff --git a/packages/worker/src/email/mailbox-inbound-ledger.ts b/packages/worker/src/email/mailbox-inbound-ledger.ts index 1f843311e5..c607316ea3 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.ts @@ -678,7 +678,7 @@ export function markMailboxInboundDeliveryReceived( ...current, state: 'received', finalizationToken: storageLease, - subscriptionEffectState: current.subscriptionEffectState ?? 'pending', + subscriptionEffectState: 'pending', usageDurationMs: current.usageDurationMs ?? usageDurationMs, usageMonth: current.usageMonth ?? usageMonth, usageBytes: current.usageBytes ?? usageBytes, From f926f5c97913fe27a47c42e4a6ec92b2637ef5bf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 16:18:53 +0000 Subject: [PATCH 54/62] feat(email): move inbound authority to Mailbox Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 66 +- ...email-inbound-mailbox-authority-mirror.sql | 121 +++ packages/worker/src/account/data-targets.ts | 1 + .../inbound-delivery-authority.node.test.ts | 230 ++++++ .../src/email/inbound-delivery-authority.ts | 748 ++++++++++++++++++ ...bound-delivery-reconciliation-authority.ts | 267 +++++++ packages/worker/src/email/inbound-effects.ts | 239 +++++- .../inbound-mailbox-mirror.workers.test.ts | 131 ++- .../src/email/inbound-mailbox.node.test.ts | 41 +- packages/worker/src/email/inbound-mailbox.ts | 24 +- packages/worker/src/email/inbound.ts | 134 ++-- .../worker/src/email/inbound.workers.test.ts | 138 ++-- packages/worker/src/email/mailbox-do.ts | 74 +- .../email/mailbox-inbound-cleanup-ledger.ts | 196 +++++ .../email/mailbox-inbound-effect-ledger.ts | 2 +- .../email/mailbox-inbound-ledger-shared.ts | 11 +- .../src/email/mailbox-inbound-ledger.ts | 48 +- .../mailbox-inbound-ledger.workers.test.ts | 46 +- .../worker/src/email/mailbox-live-mirror.ts | 13 +- packages/worker/src/email/mailbox-types.ts | 9 +- .../src/email/reconcile-inbound-deliveries.ts | 44 +- packages/worker/src/email/service.ts | 34 +- packages/worker/src/email/test-schema.ts | 42 +- tools/migration-ledger.json | 4 + 24 files changed, 2384 insertions(+), 279 deletions(-) create mode 100644 packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql create mode 100644 packages/worker/src/email/inbound-delivery-authority.node.test.ts create mode 100644 packages/worker/src/email/inbound-delivery-authority.ts create mode 100644 packages/worker/src/email/inbound-delivery-reconciliation-authority.ts create mode 100644 packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 3a5138935e..1fc27aa246 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -227,10 +227,12 @@ Durable Object export behavior: RPC; keyset pagination with prefixed cursors over those tables). Manifest counts use `countMailbox`. Phase 1 registers this consumption; phase 2 wires live dual-write for outbound terminals, provider delivery-queue graph repair, - user classification (`service.ts#setEmailMessageClassification`), and - high-risk inbound terminal paths without changing D1 authority — D1 `email_*` - rows remain the live source of truth and are still exported in the `d1` - section. See [Mailbox](#durable-objects-mailbox). + user classification (`service.ts#setEmailMessageClassification`), and user + message-graph paths. USER inbound delivery lifecycle/effect authority is now + Mailbox CAS; D1 `email_delivery_events` is its synchronous compatibility + mirror and graph-write fence. Other D1 `email_*` rows remain live graph + storage and are still exported in the `d1` section. See + [Mailbox](#durable-objects-mailbox). - `RemoteConnectorSession` exposes persisted connector metadata and tool descriptors through an export RPC. - `PackageServiceInstance` uses its status RPC as the stable persisted service @@ -712,26 +714,37 @@ owner is also used to validate canonical owner-scoped R2 keys (`emailRawMimeKey` objects install the full DDL; warm v1 objects run `CREATE INDEX IF NOT EXISTS` only. No destructive ALTERs. -**Additive inbound ledger CAS (step 2a — no authority flip):** owner-bound -atomic RPCs in `mailbox-inbound-ledger.ts` / `mailbox-inbound-effect-ledger.ts` -mirror D1 USER transitions from `inbound-delivery.ts` / `inbound-effects.ts` -(get delivery + active fingerprint window; claim/rewrite dedupe; insert charged -pending with explicit `inserted`/`existed`; claim/release storage lease; mark -rejected/received with lease/finalization CAS; prune dedupe; defer reconcile; -claim/complete usage effect; claim/complete/fail subscription effect with -retry/dead-letter/suppression; list due stale/effect work). Mutations keep -promoted columns and `detail_json` in sync and set canonical `updated_at`. -Usage/subscription complete and subscription fail require an exact -`expectedFinalizationToken` match (`event_type`/`state` = `received`); mismatch -is `lease-lost`. Storage claim clears finalization plus in-flight effect -leases/retry (and resets `processing` → `pending`) so reclaim/re-finalization -cannot be completed by a stale effect worker — stronger than D1's `json_set` -patch, required because Mailbox promotes those lease columns. The DO does -**not** perform external usage recording or subscription dispatch. Mirror -`upsertDeliveryEvent` / `upsertDeliveryEvents` remain the compatibility write -API. **These CAS RPCs are not live-wired** — D1 stays sole write authority for -inbound ledger/effects. `system:email` stays on D1 and is never written into -per-user Mailbox objects. +**USER inbound ledger authority (step 2b):** owner-bound atomic RPCs in +`mailbox-inbound-ledger.ts` / `mailbox-inbound-effect-ledger.ts` are the sole +authority for USER delivery/window/storage/rejection/receive/reconciliation and +effect transitions. Mutations keep promoted columns and `detail_json` in sync +and set canonical `updated_at`. Usage/subscription complete and subscription +fail require an exact `expectedFinalizationToken` match (`event_type`/`state` = +`received`); mismatch is `lease-lost`. Storage claim clears finalization plus +in-flight effect leases/retry (and resets `processing` → `pending`) so +reclaim/re-finalization cannot be completed by a stale effect worker. Cleanup +claim/release and orphan-cleaned tombstones are also owner-bound CAS. + +The DO does **not** perform external usage recording or subscription dispatch: +Workers claim in Mailbox, perform the D1 usage-rollup or package dispatch, then +complete/fail in Mailbox. After every successful USER CAS, +`inbound-delivery-authority.ts` synchronously upserts one full Mailbox snapshot +into D1 `email_delivery_events` (promoted columns plus `detail_json`, +owner/provider fenced, monotonic and idempotent). Pending is mirrored before +delivery reads; storing is mirrored before D1 thread/message/attachment fence +predicates. Mirror failure fails closed. Terminal/effect snapshots keep D1 +global due-owner discovery current, but D1 is never read back as ongoing +authority. + +The only reverse path is the deployment bridge: when a USER point lookup misses +in Mailbox and a pre-deploy D1 row exists, one complete D1 snapshot bootstraps +the owner-bound DO row, after which transitions continue through Mailbox CAS. +`system:email` stays on the existing D1 implementation and cannot bootstrap a +Mailbox. Migration `0129-email-inbound-mailbox-authority-mirror.sql` is +additive: it promotes compatibility/fence fields and adds the cross-store usage +effect idempotency ledger; it drops no tables. Destructive follow-up work +remains gated on the verified backup whose SHA-256 starts with `7787f8c9`; this +change is explicitly non-destructive. **Mirror write contract:** `mirrorMessage`, `upsertDeliveryEvent`, and `upsertDeliveryEvents` take complete snapshots — every persisted field is @@ -814,8 +827,9 @@ never propagate into D1 commit paths. Helpers cover full snapshots `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, `mirrorMailboxDeleteDeliveryEvent`, `mirrorMailboxDeleteThreadIfEmpty`). Partial mutation helpers are library-only; live paths prefer the graph orchestrator -below or parity purge/rebuild for deletes. D1 remains sole authority for all -live mail read/write paths. +below or parity purge/rebuild for deletes. These best-effort helpers remain for +D1-authoritative message graphs; USER inbound delivery events use the +synchronous reverse compatibility mirror described above. **Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 message graph (optional caller thread, message, attachments, then delivery diff --git a/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql b/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql new file mode 100644 index 0000000000..08b82e48fa --- /dev/null +++ b/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql @@ -0,0 +1,121 @@ +-- USER inbound delivery authority now lives in the per-owner Mailbox Durable +-- Object. These columns are a complete compatibility projection used for +-- synchronous D1 graph-write fences and global scheduled owner discovery. +-- system:email continues to use the same D1 row as its authority. + +ALTER TABLE email_delivery_events ADD COLUMN state TEXT +CHECK ( + state IS NULL + OR state IN ( + 'pending', 'storing', 'cleaning', 'received', 'rejected', 'orphan-cleaned' + ) +); +ALTER TABLE email_delivery_events ADD COLUMN fingerprint TEXT; +ALTER TABLE email_delivery_events ADD COLUMN storage_lease TEXT; +ALTER TABLE email_delivery_events ADD COLUMN storage_lease_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN cleanup_lease TEXT; +ALTER TABLE email_delivery_events ADD COLUMN cleanup_lease_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN cleanup_retry_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN expected_attachment_count INTEGER; +ALTER TABLE email_delivery_events ADD COLUMN finalization_token TEXT; +ALTER TABLE email_delivery_events ADD COLUMN reconcile_after TEXT; +ALTER TABLE email_delivery_events ADD COLUMN dedupe_expires_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN usage_effect_suppressed_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN usage_started_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN usage_effect_retry_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN usage_effect_lease TEXT; +ALTER TABLE email_delivery_events ADD COLUMN usage_effect_lease_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN subscription_effect_state TEXT +CHECK ( + subscription_effect_state IS NULL + OR subscription_effect_state IN ('pending', 'processing', 'complete', 'dead-letter') +); +ALTER TABLE email_delivery_events ADD COLUMN subscription_effect_lease TEXT; +ALTER TABLE email_delivery_events ADD COLUMN subscription_effect_lease_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN subscription_effect_retry_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN subscription_effect_attempt_count INTEGER; +ALTER TABLE email_delivery_events ADD COLUMN subscription_effect_dead_letter_at TEXT; +ALTER TABLE email_delivery_events ADD COLUMN subscription_effect_last_error TEXT; +ALTER TABLE email_delivery_events ADD COLUMN updated_at TEXT; + +UPDATE email_delivery_events +SET + state = json_extract(detail_json, '$.state'), + fingerprint = json_extract(detail_json, '$.fingerprint'), + storage_lease = json_extract(detail_json, '$.storageLease'), + storage_lease_at = json_extract(detail_json, '$.storageLeaseAt'), + cleanup_lease = json_extract(detail_json, '$.cleanupLease'), + cleanup_lease_at = json_extract(detail_json, '$.cleanupLeaseAt'), + cleanup_retry_at = json_extract(detail_json, '$.cleanupRetryAt'), + expected_attachment_count = json_extract( + detail_json, + '$.expectedAttachmentCount' + ), + finalization_token = json_extract(detail_json, '$.finalizationToken'), + reconcile_after = json_extract(detail_json, '$.reconcileAfter'), + dedupe_expires_at = json_extract(detail_json, '$.dedupeExpiresAt'), + usage_effect_suppressed_at = json_extract( + detail_json, + '$.usageEffectSuppressedAt' + ), + usage_started_at = json_extract(detail_json, '$.usageStartedAt'), + usage_effect_retry_at = json_extract(detail_json, '$.usageEffectRetryAt'), + usage_effect_lease = json_extract(detail_json, '$.usageEffectLease'), + usage_effect_lease_at = json_extract(detail_json, '$.usageEffectLeaseAt'), + subscription_effect_state = json_extract( + detail_json, + '$.subscriptionEffectState' + ), + subscription_effect_lease = json_extract( + detail_json, + '$.subscriptionEffectLease' + ), + subscription_effect_lease_at = json_extract( + detail_json, + '$.subscriptionEffectLeaseAt' + ), + subscription_effect_retry_at = json_extract( + detail_json, + '$.subscriptionEffectRetryAt' + ), + subscription_effect_attempt_count = json_extract( + detail_json, + '$.subscriptionEffectAttemptCount' + ), + subscription_effect_dead_letter_at = json_extract( + detail_json, + '$.subscriptionEffectDeadLetterAt' + ), + subscription_effect_last_error = json_extract( + detail_json, + '$.subscriptionEffectLastError' + ), + updated_at = created_at +WHERE provider IN ( + 'cloudflare-email-routing', + 'cloudflare-email-routing-dedupe' +); + +UPDATE email_delivery_events +SET updated_at = created_at +WHERE updated_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_email_delivery_events_user_state_created +ON email_delivery_events(user_id, state, created_at, id) +WHERE provider = 'cloudflare-email-routing' AND state IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_email_delivery_events_user_dedupe_expires +ON email_delivery_events(user_id, dedupe_expires_at, id) +WHERE provider = 'cloudflare-email-routing-dedupe' + AND dedupe_expires_at IS NOT NULL; + +-- Cross-store idempotency for the external D1 rollup effect. Mailbox owns the +-- effect state/lease; this ledger only prevents replay from incrementing the +-- aggregate twice if the Worker fails after D1 commit but before Mailbox CAS. +CREATE TABLE email_inbound_usage_effects ( + user_id TEXT NOT NULL, + delivery_id TEXT NOT NULL, + finalization_token TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (user_id, delivery_id, finalization_token) +); diff --git a/packages/worker/src/account/data-targets.ts b/packages/worker/src/account/data-targets.ts index b7a68442ed..dd6d580481 100644 --- a/packages/worker/src/account/data-targets.ts +++ b/packages/worker/src/account/data-targets.ts @@ -156,6 +156,7 @@ export const accountUserDataTargets: ReadonlyArray = [ { kind: 'user_id', table: 'workflow_runs' }, { kind: 'user_id', table: 'package_service_states' }, { kind: 'user_id', table: 'user_storage_buckets' }, + { kind: 'user_id', table: 'email_inbound_usage_effects' }, { kind: 'user_id', table: 'usage_rollups' }, { kind: 'user_id', table: 'feature_flag_exposure_rollups' }, { kind: 'user_id', table: 'user_activation_milestones' }, diff --git a/packages/worker/src/email/inbound-delivery-authority.node.test.ts b/packages/worker/src/email/inbound-delivery-authority.node.test.ts new file mode 100644 index 0000000000..6b4afd8e6a --- /dev/null +++ b/packages/worker/src/email/inbound-delivery-authority.node.test.ts @@ -0,0 +1,230 @@ +import { DatabaseSync } from 'node:sqlite' +import { expect, test, vi } from 'vitest' +import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' +import { buildInboundDelivery } from './inbound-delivery.ts' +import { + createUserInboundDeliveryAuthority, + mirrorUserInboundDeliverySnapshotToD1, +} from './inbound-delivery-authority.ts' +import { type MailboxInboundDeliverySnapshot } from './mailbox-inbound-ledger.ts' +import { ensureEmailTestSchema } from './test-schema.ts' + +test('UserMeter charge precedes Mailbox insert failure and replay does not charge twice', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const userId = `user-${crypto.randomUUID()}` + const now = new Date('2026-08-02T12:00:00.000Z') + const delivery = await buildInboundDelivery({ + userId, + inboxId: `inbox-${crypto.randomUUID()}`, + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'From: sender@example.com\r\n\r\nhello', + quotaDay: '2026-08-02', + now, + }) + const snapshot: MailboxInboundDeliverySnapshot = { + ...delivery, + provider: 'cloudflare-email-routing', + state: 'pending', + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + } + const order: Array = [] + let consumeCalls = 0 + let insertCalls = 0 + const meter = { + async consumeInboundDelivery() { + consumeCalls += 1 + order.push(`meter-${consumeCalls}`) + return { + outcome: 'ready' as const, + count: 1, + revision: 1, + mirrorUpdatedAt: now.toISOString(), + consumed: consumeCalls === 1, + replayed: consumeCalls > 1, + day: '2026-08-02', + resource: 'email_receives_per_day' as const, + } + }, + } + const mailbox = { + getInboundDelivery: vi.fn(async () => null), + claimInboundDeliveryWindow: vi.fn(async () => { + order.push('mailbox-window') + return snapshot + }), + insertChargedPendingInboundDelivery: vi.fn(async () => { + insertCalls += 1 + order.push(`mailbox-${insertCalls}`) + if (insertCalls === 1) throw new Error('injected Mailbox insert failure') + return { status: 'inserted' as const, delivery: snapshot } + }), + } + const namespace = (stub: object) => + ({ + idFromName: () => ({}) as DurableObjectId, + get: () => stub, + }) as unknown as DurableObjectNamespace + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace(meter), + MAILBOX: namespace(mailbox), + }, + userId, + }) + + await expect( + authority.charge({ delivery, plan: 'pro', limit: 100, now }), + ).rejects.toThrow('injected Mailbox insert failure') + expect(order).toEqual(['meter-1', 'mailbox-window', 'mailbox-1']) + + const replay = await authority.charge({ + delivery, + plan: 'pro', + limit: 100, + now, + }) + expect(replay).toBe(delivery) + expect(order).toEqual([ + 'meter-1', + 'mailbox-window', + 'mailbox-1', + 'meter-2', + 'mailbox-window', + 'mailbox-2', + ]) + expect(consumeCalls).toBe(2) + expect(insertCalls).toBe(2) + expect( + await db + .prepare( + `SELECT state, fingerprint FROM email_delivery_events + WHERE id = ? AND user_id = ?`, + ) + .bind(delivery.deliveryId, userId) + .first(), + ).toEqual({ state: 'pending', fingerprint: delivery.fingerprint }) + sqlite.close() +}) + +test('full D1 snapshot mirror cannot cross an owner or provider fence', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const now = new Date('2026-08-02T12:00:00.000Z') + const delivery = await buildInboundDelivery({ + userId: 'owner-a', + inboxId: 'inbox-a', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'hello', + quotaDay: '2026-08-02', + now, + }) + const snapshot: MailboxInboundDeliverySnapshot = { + ...delivery, + state: 'storing', + storageLease: 'lease-a', + storageLeaseAt: now.toISOString(), + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + } + await mirrorUserInboundDeliverySnapshotToD1({ + db, + userId: 'owner-a', + snapshot, + }) + await expect( + mirrorUserInboundDeliverySnapshotToD1({ + db, + userId: 'owner-b', + snapshot: { ...snapshot, storageLease: 'lease-b' }, + }), + ).rejects.toThrow('owner/provider fence') + expect( + await db + .prepare( + `SELECT user_id, state, storage_lease FROM email_delivery_events + WHERE id = ?`, + ) + .bind(delivery.deliveryId) + .first(), + ).toEqual({ + user_id: 'owner-a', + state: 'storing', + storage_lease: 'lease-a', + }) + sqlite.close() +}) + +test('stale D1 mirrors fail closed without replacing a newer Mailbox snapshot', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const userId = `user-${crypto.randomUUID()}` + const older = new Date('2026-08-02T12:00:00.000Z') + const newer = new Date('2026-08-02T12:01:00.000Z') + const delivery = await buildInboundDelivery({ + userId, + inboxId: `inbox-${crypto.randomUUID()}`, + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'hello', + quotaDay: '2026-08-02', + now: older, + }) + const received: MailboxInboundDeliverySnapshot = { + ...delivery, + state: 'received', + finalizationToken: 'final-token', + createdAt: older.toISOString(), + updatedAt: newer.toISOString(), + } + await mirrorUserInboundDeliverySnapshotToD1({ + db, + userId, + snapshot: received, + }) + await expect( + mirrorUserInboundDeliverySnapshotToD1({ + db, + userId, + snapshot: { + ...received, + state: 'storing', + storageLease: 'stale-lease', + storageLeaseAt: older.toISOString(), + updatedAt: older.toISOString(), + }, + }), + ).rejects.toThrow('owner/provider fence') + expect( + await db + .prepare( + `SELECT state, finalization_token, storage_lease + FROM email_delivery_events WHERE id = ? AND user_id = ?`, + ) + .bind(delivery.deliveryId, userId) + .first(), + ).toEqual({ + state: 'received', + finalization_token: 'final-token', + storage_lease: null, + }) + sqlite.close() +}) + +test('USER authority refuses the system email owner before bootstrap', () => { + expect(() => + createUserInboundDeliveryAuthority({ + env: {} as Parameters< + typeof createUserInboundDeliveryAuthority + >[0]['env'], + userId: 'system:email', + }), + ).toThrow('must remain in D1') +}) diff --git a/packages/worker/src/email/inbound-delivery-authority.ts b/packages/worker/src/email/inbound-delivery-authority.ts new file mode 100644 index 0000000000..c24e7732ea --- /dev/null +++ b/packages/worker/src/email/inbound-delivery-authority.ts @@ -0,0 +1,748 @@ +import { + buildEntitlementUpgradeHint, + EntitlementLimitError, +} from '#worker/entitlements/errors.ts' +import { type PlanName } from '#worker/entitlements/plans.ts' +import { + userMeterRpc, + type UserMeterEnv, +} from '#worker/entitlements/user-meter-client.ts' +import { + getInboundDelivery as getD1InboundDelivery, + getInboundDeliveryWindow as getD1InboundDeliveryWindow, + InboundDeliveryLeaseLostError, + type InboundDelivery, +} from './inbound-delivery.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { + detailJsonFromMailboxInboundSnapshot, + mailboxInboundDedupePointerId, + mailboxInboundDedupeProvider, + mailboxInboundProvider, + needsMailboxEffectReconcile, + type MailboxInboundDeliveryInsertInput, + type MailboxInboundDeliverySnapshot, +} from './mailbox-inbound-ledger.ts' +import { + type MailboxDeliveryEventInput, + type MailboxInboundDeliveryState, +} from './mailbox-types.ts' +import { systemEmailOwnerId } from './email-owner.ts' + +export type UserInboundDeliveryAuthorityEnv = { + APP_DB: D1Database +} & MailboxEnv & + UserMeterEnv + +function toInsertInput( + delivery: InboundDelivery, +): MailboxInboundDeliveryInsertInput { + return { + fingerprint: delivery.fingerprint, + deliveryId: delivery.deliveryId, + messageId: delivery.messageId, + threadId: delivery.threadId, + rawMimeKey: delivery.rawMimeKey, + inboxId: delivery.inboxId, + recipient: delivery.recipient, + envelopeFrom: delivery.envelopeFrom, + provider: delivery.provider, + quotaDay: delivery.quotaDay, + dedupeExpiresAt: delivery.dedupeExpiresAt, + usageStartedAt: delivery.usageStartedAt, + } +} + +function toInboundDelivery( + userId: string, + snapshot: MailboxInboundDeliverySnapshot, +): InboundDelivery { + const { + createdAt: _createdAt, + updatedAt: _updatedAt, + cleanupRetryAt: _cleanupRetryAt, + reconcileAfter: _reconcileAfter, + ...delivery + } = snapshot + return { ...delivery, userId } +} + +function eventTypeForState(state: MailboxInboundDeliveryState) { + switch (state) { + case 'received': + return 'received' as const + case 'rejected': + return 'rejected' as const + case 'pending': + case 'storing': + case 'cleaning': + case 'orphan-cleaned': + return 'receive_started' as const + default: { + const exhaustive: never = state + throw new Error(`Unhandled inbound delivery state: ${String(exhaustive)}`) + } + } +} + +function toMailboxBootstrapEvent(input: { + delivery: InboundDelivery + eventId: string + provider: string + createdAt: string + updatedAt: string +}): MailboxDeliveryEventInput { + const delivery = input.delivery + const detail = JSON.stringify(delivery) + return { + id: input.eventId, + messageId: delivery.state === 'received' ? delivery.messageId : null, + inboxId: delivery.inboxId, + eventType: eventTypeForState(delivery.state), + provider: input.provider, + providerMessageId: null, + providerEventId: input.eventId, + detailJson: detail, + needsEffectReconcile: + delivery.state === 'received' && needsMailboxEffectReconcile(delivery), + state: delivery.state, + fingerprint: delivery.fingerprint, + storageLease: delivery.storageLease ?? null, + storageLeaseAt: delivery.storageLeaseAt ?? null, + cleanupLease: delivery.cleanupLease ?? null, + cleanupLeaseAt: delivery.cleanupLeaseAt ?? null, + cleanupRetryAt: null, + expectedAttachmentCount: delivery.expectedAttachmentCount ?? null, + finalizationToken: delivery.finalizationToken ?? null, + reconcileAfter: null, + dedupeExpiresAt: delivery.dedupeExpiresAt, + usageEffectRecordedAt: delivery.usageEffectRecordedAt ?? null, + usageEffectSuppressedAt: delivery.usageEffectSuppressedAt ?? null, + usageStartedAt: delivery.usageStartedAt ?? null, + usageMonth: delivery.usageMonth ?? null, + usageBytes: delivery.usageBytes ?? null, + usageDurationMs: delivery.usageDurationMs ?? null, + usageEffectRetryAt: delivery.usageEffectRetryAt ?? null, + usageEffectLease: delivery.usageEffectLease ?? null, + usageEffectLeaseAt: delivery.usageEffectLeaseAt ?? null, + subscriptionEffectState: delivery.subscriptionEffectState ?? null, + subscriptionEffectLease: delivery.subscriptionEffectLease ?? null, + subscriptionEffectLeaseAt: delivery.subscriptionEffectLeaseAt ?? null, + subscriptionEffectRetryAt: delivery.subscriptionEffectRetryAt ?? null, + subscriptionEffectAttemptCount: + delivery.subscriptionEffectAttemptCount ?? null, + subscriptionEffectDeadLetterAt: + delivery.subscriptionEffectDeadLetterAt ?? null, + subscriptionEffectLastError: delivery.subscriptionEffectLastError ?? null, + createdAt: input.createdAt, + updatedAt: input.updatedAt, + } +} + +async function d1EventTimestamps(input: { + db: D1Database + userId: string + eventId: string + provider: string +}) { + return await input.db + .prepare( + `SELECT created_at, COALESCE(updated_at, created_at) AS updated_at + FROM email_delivery_events + WHERE id = ? AND user_id = ? AND provider = ? + LIMIT 1`, + ) + .bind(input.eventId, input.userId, input.provider) + .first<{ created_at: string; updated_at: string }>() +} + +/** + * Full Mailbox → D1 compatibility snapshot. The conflict fence prevents a + * different owner/provider from being overwritten, while updated_at rejects + * delayed mirrors from an older Mailbox CAS. + */ +export async function mirrorUserInboundDeliverySnapshotToD1(input: { + db: D1Database + userId: string + snapshot: MailboxInboundDeliverySnapshot + eventId?: string + provider?: string +}) { + const snapshot = input.snapshot + const eventId = input.eventId ?? snapshot.deliveryId + const provider = input.provider ?? mailboxInboundProvider + const eventType = eventTypeForState(snapshot.state) + const detailJson = detailJsonFromMailboxInboundSnapshot(snapshot, { + userId: input.userId, + }) + const result = await input.db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, state, fingerprint, + storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, + cleanup_retry_at, expected_attachment_count, finalization_token, + reconcile_after, dedupe_expires_at, usage_effect_recorded_at, + usage_effect_suppressed_at, usage_started_at, usage_month, + usage_bytes, usage_duration_ms, usage_effect_retry_at, + usage_effect_lease, usage_effect_lease_at, subscription_effect_state, + subscription_effect_lease, subscription_effect_lease_at, + subscription_effect_retry_at, subscription_effect_attempt_count, + subscription_effect_dead_letter_at, subscription_effect_last_error, + created_at, updated_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + message_id = excluded.message_id, + inbox_id = excluded.inbox_id, + event_type = excluded.event_type, + provider_event_id = excluded.provider_event_id, + detail_json = excluded.detail_json, + needs_effect_reconcile = excluded.needs_effect_reconcile, + state = excluded.state, + fingerprint = excluded.fingerprint, + storage_lease = excluded.storage_lease, + storage_lease_at = excluded.storage_lease_at, + cleanup_lease = excluded.cleanup_lease, + cleanup_lease_at = excluded.cleanup_lease_at, + cleanup_retry_at = excluded.cleanup_retry_at, + expected_attachment_count = excluded.expected_attachment_count, + finalization_token = excluded.finalization_token, + reconcile_after = excluded.reconcile_after, + dedupe_expires_at = excluded.dedupe_expires_at, + usage_effect_recorded_at = excluded.usage_effect_recorded_at, + usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, + usage_started_at = excluded.usage_started_at, + usage_month = excluded.usage_month, + usage_bytes = excluded.usage_bytes, + usage_duration_ms = excluded.usage_duration_ms, + usage_effect_retry_at = excluded.usage_effect_retry_at, + usage_effect_lease = excluded.usage_effect_lease, + usage_effect_lease_at = excluded.usage_effect_lease_at, + subscription_effect_state = excluded.subscription_effect_state, + subscription_effect_lease = excluded.subscription_effect_lease, + subscription_effect_lease_at = excluded.subscription_effect_lease_at, + subscription_effect_retry_at = excluded.subscription_effect_retry_at, + subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, + subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, + subscription_effect_last_error = excluded.subscription_effect_last_error, + updated_at = excluded.updated_at + WHERE email_delivery_events.user_id = excluded.user_id + AND email_delivery_events.provider = excluded.provider + AND excluded.updated_at >= COALESCE( + email_delivery_events.updated_at, + email_delivery_events.created_at + )`, + ) + .bind( + eventId, + snapshot.state === 'received' ? snapshot.messageId : null, + input.userId, + snapshot.inboxId, + eventType, + provider, + eventId, + detailJson, + snapshot.state === 'received' && needsMailboxEffectReconcile(snapshot) + ? 1 + : 0, + snapshot.state, + snapshot.fingerprint, + snapshot.storageLease ?? null, + snapshot.storageLeaseAt ?? null, + snapshot.cleanupLease ?? null, + snapshot.cleanupLeaseAt ?? null, + snapshot.cleanupRetryAt ?? null, + snapshot.expectedAttachmentCount ?? null, + snapshot.finalizationToken ?? null, + snapshot.reconcileAfter ?? null, + snapshot.dedupeExpiresAt, + snapshot.usageEffectRecordedAt ?? null, + snapshot.usageEffectSuppressedAt ?? null, + snapshot.usageStartedAt ?? null, + snapshot.usageMonth ?? null, + snapshot.usageBytes ?? null, + snapshot.usageDurationMs ?? null, + snapshot.usageEffectRetryAt ?? null, + snapshot.usageEffectLease ?? null, + snapshot.usageEffectLeaseAt ?? null, + snapshot.subscriptionEffectState ?? null, + snapshot.subscriptionEffectLease ?? null, + snapshot.subscriptionEffectLeaseAt ?? null, + snapshot.subscriptionEffectRetryAt ?? null, + snapshot.subscriptionEffectAttemptCount ?? null, + snapshot.subscriptionEffectDeadLetterAt ?? null, + snapshot.subscriptionEffectLastError ?? null, + snapshot.createdAt, + snapshot.updatedAt, + ) + .run() + if (Number(result.meta.changes ?? 0) < 1) { + throw new Error( + 'Mailbox inbound snapshot failed its D1 owner/provider fence.', + ) + } +} + +async function bootstrapDeliveryFromD1(input: { + env: UserInboundDeliveryAuthorityEnv + userId: string + deliveryId: string +}) { + const delivery = await getD1InboundDelivery({ + db: input.env.APP_DB, + userId: input.userId, + deliveryId: input.deliveryId, + }) + if (!delivery) return null + const timestamps = await d1EventTimestamps({ + db: input.env.APP_DB, + userId: input.userId, + eventId: input.deliveryId, + provider: mailboxInboundProvider, + }) + if (!timestamps) return null + const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) + await mailbox.upsertDeliveryEvent({ + ownerId: input.userId, + event: toMailboxBootstrapEvent({ + delivery, + eventId: input.deliveryId, + provider: mailboxInboundProvider, + createdAt: timestamps.created_at, + updatedAt: timestamps.updated_at, + }), + }) + return await mailbox.getInboundDelivery({ + ownerId: input.userId, + deliveryId: input.deliveryId, + }) +} + +async function bootstrapWindowFromD1(input: { + env: UserInboundDeliveryAuthorityEnv + userId: string + fingerprint: string + now: Date +}) { + const delivery = await getD1InboundDeliveryWindow({ + db: input.env.APP_DB, + userId: input.userId, + fingerprint: input.fingerprint, + now: input.now, + }) + if (!delivery) return null + const eventId = mailboxInboundDedupePointerId(input.fingerprint) + const timestamps = await d1EventTimestamps({ + db: input.env.APP_DB, + userId: input.userId, + eventId, + provider: mailboxInboundDedupeProvider, + }) + if (!timestamps) return null + const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) + await mailbox.upsertDeliveryEvent({ + ownerId: input.userId, + event: toMailboxBootstrapEvent({ + delivery, + eventId, + provider: mailboxInboundDedupeProvider, + createdAt: timestamps.created_at, + updatedAt: timestamps.updated_at, + }), + }) + return await mailbox.getInboundDeliveryWindow({ + ownerId: input.userId, + fingerprint: input.fingerprint, + now: input.now.toISOString(), + }) +} + +export function createUserInboundDeliveryAuthority(input: { + env: UserInboundDeliveryAuthorityEnv + userId: string +}) { + const { env, userId } = input + if (userId === systemEmailOwnerId) { + throw new Error( + 'system:email inbound delivery authority must remain in D1.', + ) + } + const mailbox = mailboxRpc({ env, userId }) + const mirror = async ( + snapshot: MailboxInboundDeliverySnapshot, + options?: { eventId?: string; provider?: string }, + ) => { + await mirrorUserInboundDeliverySnapshotToD1({ + db: env.APP_DB, + userId, + snapshot, + ...options, + }) + return toInboundDelivery(userId, snapshot) + } + return { + async get(deliveryId: string) { + const current = + (await mailbox.getInboundDelivery({ ownerId: userId, deliveryId })) ?? + (await bootstrapDeliveryFromD1({ env, userId, deliveryId })) + return current ? await mirror(current) : null + }, + async getWindow(fingerprint: string, now: Date) { + const current = + (await mailbox.getInboundDeliveryWindow({ + ownerId: userId, + fingerprint, + now: now.toISOString(), + })) ?? (await bootstrapWindowFromD1({ env, userId, fingerprint, now })) + return current + ? await mirror(current, { + eventId: mailboxInboundDedupePointerId(fingerprint), + provider: mailboxInboundDedupeProvider, + }) + : null + }, + async claimWindow(delivery: InboundDelivery, now: Date) { + const snapshot = await mailbox.claimInboundDeliveryWindow({ + ownerId: userId, + delivery: toInsertInput(delivery), + now: now.toISOString(), + }) + return await mirror(snapshot, { + eventId: mailboxInboundDedupePointerId(delivery.fingerprint), + provider: mailboxInboundDedupeProvider, + }) + }, + async charge(input: { + delivery: InboundDelivery + plan: PlanName + limit: number + now: Date + }) { + const existing = await this.get(input.delivery.deliveryId) + if (existing) return existing + const updatedAt = input.now.toISOString() + const meter = userMeterRpc({ env, userId }) + let meterResult = await meter.consumeInboundDelivery({ + deliveryId: input.delivery.deliveryId, + resource: 'email_receives_per_day', + day: input.delivery.quotaDay, + limit: input.limit, + updatedAt, + }) + if (meterResult.outcome === 'needs_bootstrap') { + await meter.initialize({ + resource: 'email_receives_per_day', + day: input.delivery.quotaDay, + count: 0, + updatedAt, + }) + meterResult = await meter.consumeInboundDelivery({ + deliveryId: input.delivery.deliveryId, + resource: 'email_receives_per_day', + day: input.delivery.quotaDay, + limit: input.limit, + updatedAt, + }) + if (meterResult.outcome === 'needs_bootstrap') { + throw new Error( + 'UserMeter inbound delivery consume still needs bootstrap after initialize.', + ) + } + } + if (!meterResult.consumed && !meterResult.replayed) { + throw new EntitlementLimitError({ + resource: 'email_receives_per_day', + plan: input.plan, + limit: input.limit, + current: meterResult.count, + upgradeHint: buildEntitlementUpgradeHint('email_receives_per_day'), + }) + } + // UserMeter is deliberately the first durable mutation. Only after + // its idempotent claim succeeds may Mailbox claim the dedupe window + // and insert the charged delivery. + const claimedDelivery = await this.claimWindow(input.delivery, input.now) + const result = await mailbox.insertChargedPendingInboundDelivery({ + ownerId: userId, + delivery: toInsertInput(claimedDelivery), + now: updatedAt, + }) + await mirror(result.delivery) + return result.status === 'inserted' && + claimedDelivery.deliveryId === input.delivery.deliveryId + ? input.delivery + : toInboundDelivery(userId, result.delivery) + }, + async claimStorage( + delivery: InboundDelivery, + expectedAttachmentCount: number, + usageStartedAt?: string, + now = new Date(), + ) { + const result = await mailbox.claimInboundDeliveryStorage({ + ownerId: userId, + deliveryId: delivery.deliveryId, + expectedAttachmentCount, + usageStartedAt, + now: now.toISOString(), + }) + if (result.status === 'claimed') { + return { + claimed: true as const, + delivery: await mirror(result.delivery), + } + } + return { + claimed: false as const, + delivery: result.delivery ? await mirror(result.delivery) : null, + } + }, + async releaseStorage(delivery: InboundDelivery, now = new Date()) { + if (!delivery.storageLease) return + const result = await mailbox.releaseInboundDeliveryStorage({ + ownerId: userId, + deliveryId: delivery.deliveryId, + storageLease: delivery.storageLease, + now: now.toISOString(), + }) + if (result.status === 'released') await mirror(result.delivery) + }, + async reject(delivery: InboundDelivery, reason: string, now = new Date()) { + const result = await mailbox.markInboundDeliveryRejected({ + ownerId: userId, + deliveryId: delivery.deliveryId, + reason, + expectedStorageLease: delivery.storageLease, + expectedState: delivery.state, + now: now.toISOString(), + }) + if ( + result.status === 'rejected' || + result.status === 'already-rejected' + ) { + await mirror(result.delivery) + return true + } + if (result.status === 'already-received') { + await mirror(result.delivery) + return false + } + throw new InboundDeliveryLeaseLostError( + 'Inbound rejection lost a state race; delivery should be retried.', + ) + }, + async receive(input: { + delivery: InboundDelivery + usageDurationMs: number + usageMonth: string + usageBytes: number + now?: Date + }) { + if (!input.delivery.storageLease) { + throw new InboundDeliveryLeaseLostError( + 'Inbound delivery finalization requires a storage lease.', + ) + } + const result = await mailbox.markInboundDeliveryReceived({ + ownerId: userId, + deliveryId: input.delivery.deliveryId, + storageLease: input.delivery.storageLease, + usageDurationMs: input.usageDurationMs, + usageMonth: input.usageMonth, + usageBytes: input.usageBytes, + now: (input.now ?? new Date()).toISOString(), + }) + if ( + result.status === 'received' || + result.status === 'already-received' + ) { + return await mirror(result.delivery) + } + throw new InboundDeliveryLeaseLostError( + 'Inbound delivery storage lease was lost before finalization.', + ) + }, + async deferReconciliation(deliveryId: string, now = new Date()) { + const result = await mailbox.deferInboundDeliveryReconciliation({ + ownerId: userId, + deliveryId, + now: now.toISOString(), + }) + if (result.status === 'deferred') await mirror(result.delivery) + return result + }, + async listDueStale(now = new Date(), limit?: number) { + return await mailbox.listDueStaleInboundDeliveries({ + ownerId: userId, + now: now.toISOString(), + limit, + }) + }, + async claimCleanup(deliveryId: string, now = new Date()) { + const result = await mailbox.claimInboundDeliveryCleanup({ + ownerId: userId, + deliveryId, + now: now.toISOString(), + }) + if (result.delivery) await mirror(result.delivery) + return result + }, + async releaseCleanup( + deliveryId: string, + cleanupLease: string, + now = new Date(), + ) { + const result = await mailbox.releaseInboundDeliveryCleanup({ + ownerId: userId, + deliveryId, + cleanupLease, + now: now.toISOString(), + }) + if (result.status === 'released') await mirror(result.delivery) + return result + }, + async markOrphanCleaned(input: { + deliveryId: string + cleanupLease: string + outcome: 'deleted' | 'delete-failed' + now?: Date + }) { + const result = await mailbox.markInboundDeliveryOrphanCleaned({ + ownerId: userId, + deliveryId: input.deliveryId, + cleanupLease: input.cleanupLease, + outcome: input.outcome, + now: (input.now ?? new Date()).toISOString(), + }) + if (result.status === 'orphan-cleaned') await mirror(result.delivery) + return result + }, + async pruneExpiredDedupe(now = new Date(), limit?: number) { + const result = await mailbox.pruneExpiredInboundDedupePointers({ + ownerId: userId, + now: now.toISOString(), + limit, + }) + await env.APP_DB.prepare( + `DELETE FROM email_delivery_events + WHERE id IN ( + SELECT id FROM email_delivery_events + WHERE user_id = ? + AND provider = ? + AND dedupe_expires_at <= ? + ORDER BY created_at ASC, id ASC + LIMIT ? + )`, + ) + .bind( + userId, + mailboxInboundDedupeProvider, + now.toISOString(), + limit ?? 20, + ) + .run() + return result.pruned + }, + async claimUsageEffect(input: { + deliveryId: string + expectedFinalizationToken?: string + now?: Date + }) { + const result = await mailbox.claimInboundUsageEffect({ + ownerId: userId, + deliveryId: input.deliveryId, + expectedFinalizationToken: input.expectedFinalizationToken, + now: (input.now ?? new Date()).toISOString(), + }) + if (result.delivery) await mirror(result.delivery) + return result + }, + async completeUsageEffect(input: { + deliveryId: string + usageEffectLease: string + expectedFinalizationToken: string + mode: 'recorded' | 'suppressed' + usageMonth: string + usageBytes: number + usageDurationMs: number + now?: Date + }) { + const result = await mailbox.completeInboundUsageEffect({ + ownerId: userId, + deliveryId: input.deliveryId, + usageEffectLease: input.usageEffectLease, + expectedFinalizationToken: input.expectedFinalizationToken, + mode: input.mode, + usageMonth: input.usageMonth, + usageBytes: input.usageBytes, + usageDurationMs: input.usageDurationMs, + now: (input.now ?? new Date()).toISOString(), + }) + if (result.status !== 'lease-lost') await mirror(result.delivery) + return result + }, + async claimSubscriptionEffect(input: { + deliveryId: string + expectedFinalizationToken?: string + now?: Date + }) { + const result = await mailbox.claimInboundSubscriptionEffect({ + ownerId: userId, + deliveryId: input.deliveryId, + expectedFinalizationToken: input.expectedFinalizationToken, + now: (input.now ?? new Date()).toISOString(), + }) + if (result.delivery) await mirror(result.delivery) + return result + }, + async completeSubscriptionEffect(input: { + deliveryId: string + subscriptionEffectLease: string + expectedFinalizationToken: string + mode: 'complete' | 'suppressed' + suppressionReason?: string + now?: Date + }) { + const result = await mailbox.completeInboundSubscriptionEffect({ + ownerId: userId, + deliveryId: input.deliveryId, + subscriptionEffectLease: input.subscriptionEffectLease, + expectedFinalizationToken: input.expectedFinalizationToken, + mode: input.mode, + suppressionReason: input.suppressionReason, + now: (input.now ?? new Date()).toISOString(), + }) + if (result.status !== 'lease-lost') await mirror(result.delivery) + return result + }, + async failSubscriptionEffect(input: { + deliveryId: string + subscriptionEffectLease: string + expectedFinalizationToken: string + error: string + now?: Date + }) { + const result = await mailbox.failInboundSubscriptionEffect({ + ownerId: userId, + deliveryId: input.deliveryId, + subscriptionEffectLease: input.subscriptionEffectLease, + expectedFinalizationToken: input.expectedFinalizationToken, + error: input.error, + now: (input.now ?? new Date()).toISOString(), + }) + if (result.status !== 'lease-lost') await mirror(result.delivery) + return result + }, + async listDueEffects(now = new Date(), limit?: number) { + return await mailbox.listDueInboundEffectWork({ + ownerId: userId, + now: now.toISOString(), + limit, + }) + }, + } +} diff --git a/packages/worker/src/email/inbound-delivery-reconciliation-authority.ts b/packages/worker/src/email/inbound-delivery-reconciliation-authority.ts new file mode 100644 index 0000000000..4c4eafabbe --- /dev/null +++ b/packages/worker/src/email/inbound-delivery-reconciliation-authority.ts @@ -0,0 +1,267 @@ +import PostalMime from 'postal-mime' +import { emailRawMimeKey } from './blob-keys.ts' +import { + createUserInboundDeliveryAuthority, + type UserInboundDeliveryAuthorityEnv, +} from './inbound-delivery-authority.ts' +import { + InboundDeliveryLeaseLostError, + type InboundDelivery, +} from './inbound-delivery.ts' +import { + insertEmailAttachments, + getEmailMessageById, + listEmailAttachmentsForMessage, +} from './repo.ts' + +const staleBatchSize = 20 + +function parsedAttachmentSize( + content: string | ArrayBuffer | Uint8Array, +) { + return typeof content === 'string' + ? new TextEncoder().encode(content).byteLength + : content.byteLength +} + +async function bootstrapPreDeployDueRows(input: { + env: UserInboundDeliveryAuthorityEnv + userId: string + now: Date +}) { + const rows = await input.env.APP_DB.prepare( + `SELECT id + FROM email_delivery_events + WHERE user_id = ? + AND provider = 'cloudflare-email-routing' + AND event_type = 'receive_started' + AND created_at < ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + new Date(input.now.getTime() - 48 * 60 * 60 * 1000).toISOString(), + staleBatchSize, + ) + .all<{ id: string }>() + const authority = createUserInboundDeliveryAuthority(input) + for (const row of rows.results ?? []) await authority.get(row.id) +} + +async function recoverCommittedDelivery(input: { + env: UserInboundDeliveryAuthorityEnv & { EMAIL_BLOBS: R2Bucket } + userId: string + deliveryId: string + now: Date +}) { + const authority = createUserInboundDeliveryAuthority(input) + const delivery = await authority.get(input.deliveryId) + if (!delivery) return false + const message = await getEmailMessageById({ + db: input.env.APP_DB, + userId: input.userId, + messageId: delivery.messageId, + }) + if (!message?.rawMimeKey) return false + if ( + message.rawMimeKey !== emailRawMimeKey(input.userId, delivery.messageId) + ) { + throw new Error( + 'Inbound message raw MIME key is outside its user namespace.', + ) + } + const object = await input.env.EMAIL_BLOBS.get(message.rawMimeKey) + if (!object) return false + const parsed = await PostalMime.parse(await object.text(), { + attachmentEncoding: 'arraybuffer', + }) + const claim = await authority.claimStorage( + delivery, + parsed.attachments.length, + delivery.usageStartedAt, + input.now, + ) + if (!claim.claimed) return claim.delivery?.state === 'received' + const storageLease = claim.delivery.storageLease + if (!storageLease) { + throw new InboundDeliveryLeaseLostError( + 'Recovered inbound delivery has no storage lease.', + ) + } + try { + await insertEmailAttachments({ + db: input.env.APP_DB, + messageId: message.id, + ignoreConflicts: true, + inboundDeliveryFence: { + deliveryId: claim.delivery.deliveryId, + userId: input.userId, + storageLease, + }, + attachments: parsed.attachments.map((attachment, index) => ({ + id: `${message.id}:attachment:${index}`, + filename: attachment.filename, + contentType: attachment.mimeType, + contentId: attachment.contentId ?? null, + disposition: attachment.disposition, + size: parsedAttachmentSize(attachment.content), + storageKind: 'raw-mime', + storageKey: null, + })), + }) + const attachments = await listEmailAttachmentsForMessage({ + db: input.env.APP_DB, + messageId: message.id, + }) + if (attachments.length !== parsed.attachments.length) { + throw new InboundDeliveryLeaseLostError( + 'Inbound attachment recovery did not commit every attachment.', + ) + } + await authority.receive({ + delivery: claim.delivery, + usageDurationMs: claim.delivery.usageStartedAt + ? input.now.getTime() - Date.parse(claim.delivery.usageStartedAt) + : 0, + usageMonth: (message.receivedAt ?? message.createdAt).slice(0, 7), + usageBytes: message.rawSize ?? 0, + now: input.now, + }) + return true + } catch (error) { + await authority + .releaseStorage(claim.delivery, input.now) + .catch(() => undefined) + throw error + } +} + +export async function reconcileUserStaleInboundDeliveries(input: { + env: UserInboundDeliveryAuthorityEnv & { EMAIL_BLOBS: R2Bucket } + userId: string + now?: Date + deadlineMs?: number +}) { + const now = input.now ?? new Date() + await bootstrapPreDeployDueRows({ + env: input.env, + userId: input.userId, + now, + }) + const authority = createUserInboundDeliveryAuthority(input) + const due = await authority.listDueStale(now, staleBatchSize) + let recovered = 0 + let cleaned = 0 + let budgetExhausted = false + for (const snapshot of due.deliveries) { + if (input.deadlineMs != null && Date.now() >= input.deadlineMs) { + budgetExhausted = true + break + } + const message = await getEmailMessageById({ + db: input.env.APP_DB, + userId: input.userId, + messageId: snapshot.messageId, + }) + if (message) { + try { + if ( + await recoverCommittedDelivery({ + env: input.env, + userId: input.userId, + deliveryId: snapshot.deliveryId, + now, + }) + ) { + recovered += 1 + } else { + await authority.deferReconciliation(snapshot.deliveryId, now) + } + } catch (error) { + console.warn( + 'inbound-email-partial-delivery-recovery-failed', + snapshot.deliveryId, + error, + ) + await authority.deferReconciliation(snapshot.deliveryId, now) + } + continue + } + + const claim = await authority.claimCleanup(snapshot.deliveryId, now) + if (claim.status !== 'claimed') continue + const racedMessage = await getEmailMessageById({ + db: input.env.APP_DB, + userId: input.userId, + messageId: snapshot.messageId, + }) + if (racedMessage) { + await authority.releaseCleanup( + snapshot.deliveryId, + claim.delivery.cleanupLease!, + now, + ) + continue + } + let outcome: 'deleted' | 'delete-failed' = 'deleted' + try { + await input.env.EMAIL_BLOBS.delete(snapshot.rawMimeKey) + } catch (error) { + outcome = 'delete-failed' + console.warn( + 'inbound-email-orphan-blob-delete-failed', + snapshot.rawMimeKey, + error, + ) + } + const finalized = await authority.markOrphanCleaned({ + deliveryId: snapshot.deliveryId, + cleanupLease: claim.delivery.cleanupLease!, + outcome, + now, + }) + if (finalized.status === 'orphan-cleaned') { + if (outcome === 'deleted') cleaned += 1 + } + } + return { + recovered, + cleaned, + ...(budgetExhausted ? { budgetExhausted: true as const } : {}), + } +} + +export async function pruneUserExpiredInboundDedupePointers(input: { + env: UserInboundDeliveryAuthorityEnv + userId: string + now?: Date + limit?: number +}) { + const now = input.now ?? new Date() + const authority = createUserInboundDeliveryAuthority(input) + const bridgeRows = await input.env.APP_DB.prepare( + `SELECT detail_json + FROM email_delivery_events + WHERE user_id = ? + AND provider = ? + AND COALESCE( + dedupe_expires_at, + json_extract(detail_json, '$.dedupeExpiresAt') + ) <= ? + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind( + input.userId, + 'cloudflare-email-routing-dedupe', + now.toISOString(), + input.limit ?? 20, + ) + .all<{ detail_json: string }>() + for (const row of bridgeRows.results ?? []) { + const delivery = JSON.parse(row.detail_json) as InboundDelivery + await authority.claimWindow(delivery, now) + } + return await authority.pruneExpiredDedupe(now, input.limit) +} diff --git a/packages/worker/src/email/inbound-effects.ts b/packages/worker/src/email/inbound-effects.ts index 4bda1b0f46..a273b699a1 100644 --- a/packages/worker/src/email/inbound-effects.ts +++ b/packages/worker/src/email/inbound-effects.ts @@ -1,5 +1,6 @@ import { getInboundDelivery } from './inbound-delivery.ts' import { withAccountWriteLease } from '#worker/account/deletion-state.ts' +import { createUserInboundDeliveryAuthority } from './inbound-delivery-authority.ts' import { dispatchInboundEmailSubscriptionEvents, dispatchSystemInboundEmailSubscriptionEvents, @@ -21,7 +22,12 @@ export function resolveSubscriptionEffectFailure(attemptCount: number) { type InboundEffectsEnv = Pick< Env, - 'APP_DB' | 'BUNDLE_ARTIFACTS_KV' | 'APP_BASE_URL' | 'USAGE_EVENTS' + | 'APP_DB' + | 'BUNDLE_ARTIFACTS_KV' + | 'APP_BASE_URL' + | 'USAGE_EVENTS' + | 'MAILBOX' + | 'USER_METER' > async function recordInboundUsageEffect(input: { @@ -232,6 +238,193 @@ async function recordInboundUsageEffect(input: { .run() } +async function recordUserInboundUsageRollup(input: { + env: InboundEffectsEnv + userId: string + deliveryId: string + finalizationToken: string + usageMonth: string + usageBytes: number + usageDurationMs: number + now: Date +}) { + if (input.env.USAGE_EVENTS) return + try { + await input.env.APP_DB.batch([ + input.env.APP_DB.prepare( + `INSERT INTO usage_rollups ( + user_id, metric, month, event_count, error_count, + total_duration_ms, total_cpu_ms, total_bytes, updated_at + ) + SELECT ?, 'email_received', ?, 1, 0, ?, 0, ?, ? + WHERE NOT EXISTS ( + SELECT 1 FROM email_inbound_usage_effects + WHERE user_id = ? AND delivery_id = ? AND finalization_token = ? + ) + ON CONFLICT (user_id, metric, month) DO UPDATE SET + event_count = event_count + 1, + total_duration_ms = total_duration_ms + excluded.total_duration_ms, + total_bytes = total_bytes + excluded.total_bytes, + updated_at = excluded.updated_at`, + ).bind( + input.userId, + input.usageMonth, + Math.round(input.usageDurationMs), + Math.round(input.usageBytes), + input.now.toISOString(), + input.userId, + input.deliveryId, + input.finalizationToken, + ), + input.env.APP_DB.prepare( + `INSERT OR IGNORE INTO email_inbound_usage_effects ( + user_id, delivery_id, finalization_token, created_at + ) VALUES (?, ?, ?, ?)`, + ).bind( + input.userId, + input.deliveryId, + input.finalizationToken, + input.now.toISOString(), + ), + ]) + } catch (error) { + if ( + !(error instanceof Error) || + !error.message.includes('no such table: usage_rollups') + ) { + throw error + } + } +} + +async function processUserInboundDeliveryEffectsWithLeaseHeld(input: { + env: InboundEffectsEnv + userId: string + deliveryId: string + expectedFinalizationToken?: string + durationMs?: number + now?: Date + waitUntil?: (promise: Promise) => void +}) { + const now = input.now ?? new Date() + const authority = createUserInboundDeliveryAuthority({ + env: input.env, + userId: input.userId, + }) + const delivery = await authority.get(input.deliveryId) + if ( + delivery?.state !== 'received' || + !delivery.finalizationToken || + (input.expectedFinalizationToken != null && + delivery.finalizationToken !== input.expectedFinalizationToken) + ) { + return { outcome: 'stale' as const } + } + const message = await getEmailMessageById({ + db: input.env.APP_DB, + userId: input.userId, + messageId: delivery.messageId, + }) + const usageMonth = + delivery.usageMonth ?? + (message + ? (message.receivedAt ?? message.createdAt).slice(0, 7) + : now.toISOString().slice(0, 7)) + const usageBytes = delivery.usageBytes ?? message?.rawSize ?? 0 + const usageDurationMs = delivery.usageDurationMs ?? input.durationMs ?? 0 + const usageClaim = await authority.claimUsageEffect({ + deliveryId: delivery.deliveryId, + expectedFinalizationToken: delivery.finalizationToken, + now, + }) + if (usageClaim.status === 'claimed') { + await recordUserInboundUsageRollup({ + env: input.env, + userId: input.userId, + deliveryId: delivery.deliveryId, + finalizationToken: delivery.finalizationToken, + usageMonth, + usageBytes, + usageDurationMs, + now, + }) + const completed = await authority.completeUsageEffect({ + deliveryId: delivery.deliveryId, + usageEffectLease: usageClaim.delivery.usageEffectLease!, + expectedFinalizationToken: delivery.finalizationToken, + mode: 'recorded', + usageMonth, + usageBytes, + usageDurationMs, + now, + }) + if (completed.status === 'lease-lost') { + return { outcome: 'stale' as const } + } + } + + const subscriptionClaim = await authority.claimSubscriptionEffect({ + deliveryId: delivery.deliveryId, + expectedFinalizationToken: delivery.finalizationToken, + now, + }) + if (subscriptionClaim.status !== 'claimed') { + return { outcome: 'usage-only' as const } + } + const effectLease = subscriptionClaim.delivery.subscriptionEffectLease! + try { + if (!message) { + await authority.completeSubscriptionEffect({ + deliveryId: delivery.deliveryId, + subscriptionEffectLease: effectLease, + expectedFinalizationToken: delivery.finalizationToken, + mode: 'suppressed', + suppressionReason: 'missing-message', + now, + }) + return { outcome: 'missing-message' as const } + } + await dispatchInboundEmailSubscriptionEvents({ + env: input.env, + userId: input.userId, + message, + waitUntil: input.waitUntil, + }) + const completed = await authority.completeSubscriptionEffect({ + deliveryId: delivery.deliveryId, + subscriptionEffectLease: effectLease, + expectedFinalizationToken: delivery.finalizationToken, + mode: 'complete', + now, + }) + if (completed.status === 'lease-lost') { + return { outcome: 'stale' as const } + } + return { outcome: 'complete' as const } + } catch (error) { + const failure = await authority.failSubscriptionEffect({ + deliveryId: delivery.deliveryId, + subscriptionEffectLease: effectLease, + expectedFinalizationToken: delivery.finalizationToken, + error: error instanceof Error ? error.message : String(error), + now, + }) + if (failure.status === 'lease-lost') { + return { outcome: 'stale' as const } + } + if (failure.status === 'dead-letter') { + console.error('inbound-email-subscription-effect-dead-lettered', { + userId: input.userId, + deliveryId: delivery.deliveryId, + attemptCount: failure.delivery.subscriptionEffectAttemptCount, + error, + }) + return { outcome: 'dead-letter' as const } + } + throw error + } +} + async function processInboundDeliveryEffectsWithLeaseHeld(input: { env: InboundEffectsEnv userId: string @@ -513,7 +706,8 @@ export async function processInboundDeliveryEffects( return await withAccountWriteLease({ db: input.env.APP_DB, stableUserId: input.userId, - write: async () => await processInboundDeliveryEffectsWithLeaseHeld(input), + write: async () => + await processUserInboundDeliveryEffectsWithLeaseHeld(input), }) } @@ -524,6 +718,47 @@ export async function reconcileInboundDeliveryEffectsForUser(input: { limit?: number }) { const now = input.now ?? new Date() + if (input.userId !== systemEmailOwnerId) { + const authority = createUserInboundDeliveryAuthority({ + env: input.env, + userId: input.userId, + }) + const bridgeRows = await input.env.APP_DB.prepare( + `SELECT id FROM email_delivery_events + WHERE user_id = ? + AND provider = 'cloudflare-email-routing' + AND event_type = 'received' + AND needs_effect_reconcile = 1 + ORDER BY created_at ASC, id ASC + LIMIT ?`, + ) + .bind(input.userId, input.limit ?? 20) + .all<{ id: string }>() + for (const row of bridgeRows.results ?? []) await authority.get(row.id) + const due = await authority.listDueEffects(now, input.limit) + let processed = 0 + let errors = 0 + for (const delivery of due.deliveries) { + try { + await processInboundDeliveryEffects({ + env: input.env, + userId: input.userId, + deliveryId: delivery.deliveryId, + now: input.now, + }) + processed += 1 + } catch (error) { + errors += 1 + console.warn( + 'inbound-email-effect-reconciliation-failed', + input.userId, + delivery.deliveryId, + error, + ) + } + } + return { processed, errors } + } const leaseExpiredBefore = new Date( now.getTime() - subscriptionEffectLeaseMs, ).toISOString() diff --git a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts index 9bb2f63c29..6d02302908 100644 --- a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts +++ b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts @@ -90,7 +90,89 @@ function createFailingEmailBlobs() { const inboundMailboxMirrorTimeoutMs = 30_000 -/** MAILBOX stub that fails or hangs every mirror RPC without touching the real DO. */ +function createLedgerBackedMailboxStub( + mailbox: ReturnType, +) { + return { + async getInboundDelivery( + ...args: Parameters + ) { + return await mailbox.getInboundDelivery(...args) + }, + async getInboundDeliveryWindow( + ...args: Parameters + ) { + return await mailbox.getInboundDeliveryWindow(...args) + }, + async claimInboundDeliveryWindow( + ...args: Parameters + ) { + return await mailbox.claimInboundDeliveryWindow(...args) + }, + async insertChargedPendingInboundDelivery( + ...args: Parameters + ) { + return await mailbox.insertChargedPendingInboundDelivery(...args) + }, + async claimInboundDeliveryStorage( + ...args: Parameters + ) { + return await mailbox.claimInboundDeliveryStorage(...args) + }, + async releaseInboundDeliveryStorage( + ...args: Parameters + ) { + return await mailbox.releaseInboundDeliveryStorage(...args) + }, + async markInboundDeliveryRejected( + ...args: Parameters + ) { + return await mailbox.markInboundDeliveryRejected(...args) + }, + async markInboundDeliveryReceived( + ...args: Parameters + ) { + return await mailbox.markInboundDeliveryReceived(...args) + }, + async pruneExpiredInboundDedupePointers( + ...args: Parameters + ) { + return await mailbox.pruneExpiredInboundDedupePointers(...args) + }, + async listDueStaleInboundDeliveries( + ...args: Parameters + ) { + return await mailbox.listDueStaleInboundDeliveries(...args) + }, + async claimInboundUsageEffect( + ...args: Parameters + ) { + return await mailbox.claimInboundUsageEffect(...args) + }, + async completeInboundUsageEffect( + ...args: Parameters + ) { + return await mailbox.completeInboundUsageEffect(...args) + }, + async claimInboundSubscriptionEffect( + ...args: Parameters + ) { + return await mailbox.claimInboundSubscriptionEffect(...args) + }, + async completeInboundSubscriptionEffect( + ...args: Parameters + ) { + return await mailbox.completeInboundSubscriptionEffect(...args) + }, + async failInboundSubscriptionEffect( + ...args: Parameters + ) { + return await mailbox.failInboundSubscriptionEffect(...args) + }, + } +} + +/** MAILBOX namespace that fails graph-mirror RPCs but keeps ledger CAS real. */ function createInboundMailboxStubEnv(input: { mode: 'throw' | 'hang' base?: typeof env @@ -101,23 +183,26 @@ function createInboundMailboxStubEnv(input: { throw new Error('simulated mailbox failure') } const method = input.mode === 'hang' ? hang : fail - const stub = { - mirrorMessage: method, - upsertDeliveryEvent: method, - upsertDeliveryEvents: method, - touchThread: method, - updateMessageDelivery: method, - setMessageClassification: method, - deleteMessageMetadata: method, - deleteDeliveryEvent: method, - deleteThreadIfEmpty: method, - } return { ...base, APP_BASE_URL: platformBaseUrl, MAILBOX: { idFromName: (name: string) => base.MAILBOX.idFromName(name), - get: () => stub, + get: (id: DurableObjectId) => { + const mailbox = base.MAILBOX.get(id) + return { + ...createLedgerBackedMailboxStub(mailbox), + mirrorMessage: method, + upsertDeliveryEvent: method, + upsertDeliveryEvents: method, + touchThread: method, + updateMessageDelivery: method, + setMessageClassification: method, + deleteMessageMetadata: method, + deleteDeliveryEvent: method, + deleteThreadIfEmpty: method, + } + }, } as unknown as DurableObjectNamespace, } } @@ -257,24 +342,12 @@ test( }) const order: Array = [] const delayedStub = { + ...createLedgerBackedMailboxStub(real), async mirrorMessage(...args: Parameters) { order.push('graph-message') await graphGate return await real.mirrorMessage(...args) }, - async upsertDeliveryEvents( - ...args: Parameters - ) { - order.push('graph-batch') - await graphGate - return await real.upsertDeliveryEvents(...args) - }, - async upsertDeliveryEvent( - ...args: Parameters - ) { - order.push('post-effects-event') - return await real.upsertDeliveryEvent(...args) - }, } const orderedEnv = { @@ -334,11 +407,7 @@ test( releaseGraph() await drainWaitUntil(waitUntilPromises) - expect(order).toEqual([ - 'graph-message', - 'graph-batch', - 'post-effects-event', - ]) + expect(order).toEqual(['graph-message']) const mailbox = rpcFor(userId) const mirroredEvents = await mailbox.listDeliveryEvents({ diff --git a/packages/worker/src/email/inbound-mailbox.node.test.ts b/packages/worker/src/email/inbound-mailbox.node.test.ts index 925a1c524e..53caf0c396 100644 --- a/packages/worker/src/email/inbound-mailbox.node.test.ts +++ b/packages/worker/src/email/inbound-mailbox.node.test.ts @@ -9,9 +9,7 @@ const mocks = vi.hoisted(() => ({ events: [], eventsTruncated: false, })), - mirrorMailboxDeliveryEventFromD1: vi.fn(async () => ({ - status: 'mirrored' as const, - })), + authorityGet: vi.fn(async () => ({ state: 'received' as const })), processInboundDeliveryEffects: vi.fn(async () => ({ outcome: 'complete' as const, })), @@ -19,7 +17,10 @@ const mocks = vi.hoisted(() => ({ vi.mock('./mailbox-live-mirror.ts', () => ({ mirrorMailboxMessageGraphFromD1: mocks.mirrorMailboxMessageGraphFromD1, - mirrorMailboxDeliveryEventFromD1: mocks.mirrorMailboxDeliveryEventFromD1, +})) + +vi.mock('./inbound-delivery-authority.ts', () => ({ + createUserInboundDeliveryAuthority: () => ({ get: mocks.authorityGet }), })) vi.mock('./inbound-effects.ts', () => ({ @@ -44,7 +45,7 @@ function createCapturedWaitUntilContext() { function resetMocks() { mocks.mirrorMailboxMessageGraphFromD1.mockReset() - mocks.mirrorMailboxDeliveryEventFromD1.mockReset() + mocks.authorityGet.mockReset() mocks.processInboundDeliveryEffects.mockReset() mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue({ messageId: 'msg-1', @@ -52,15 +53,13 @@ function resetMocks() { events: [], eventsTruncated: false, }) - mocks.mirrorMailboxDeliveryEventFromD1.mockResolvedValue({ - status: 'mirrored' as const, - }) + mocks.authorityGet.mockResolvedValue({ state: 'received' as const }) mocks.processInboundDeliveryEffects.mockResolvedValue({ outcome: 'complete' as const, }) } -test('received terminal work orders graph then effects then event mirror via waitUntil', async () => { +test('received terminal work orders graph then effects then authority snapshot via waitUntil', async () => { resetMocks() consoleError.mockImplementation(() => {}) const order: Array = [] @@ -84,9 +83,9 @@ test('received terminal work orders graph then effects then event mirror via wai order.push('effects') return { outcome: 'complete' as const } }) - mocks.mirrorMailboxDeliveryEventFromD1.mockImplementation(async () => { + mocks.authorityGet.mockImplementation(async () => { order.push('event') - return { status: 'mirrored' as const } + return { state: 'received' as const } }) const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() @@ -121,12 +120,7 @@ test('received terminal work orders graph then effects then event mirror via wai durationMs: 12, waitUntil: expect.any(Function), }) - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith({ - env, - db: env.APP_DB, - userId: 'user-aaa', - eventId: 'delivery-1', - }) + expect(mocks.authorityGet).toHaveBeenCalledWith('delivery-1') }) test('graph failure does not skip D1 effects; effects failure skips event mirror and logs once', async () => { @@ -154,7 +148,7 @@ test('graph failure does not skip D1 effects; effects failure skips event mirror }) expect(mocks.processInboundDeliveryEffects).toHaveBeenCalledTimes(1) - expect(mocks.mirrorMailboxDeliveryEventFromD1).not.toHaveBeenCalled() + expect(mocks.authorityGet).not.toHaveBeenCalled() expect(consoleError).toHaveBeenCalledWith( 'Inbound email effect dispatch failed', expect.objectContaining({ message: 'effects exploded' }), @@ -182,10 +176,10 @@ test('received and rejected coordinators skip system:email owners', async () => expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() expect(mocks.processInboundDeliveryEffects).not.toHaveBeenCalled() - expect(mocks.mirrorMailboxDeliveryEventFromD1).not.toHaveBeenCalled() + expect(mocks.authorityGet).not.toHaveBeenCalled() }) -test('rejected terminal schedules delivery-event mirror only', async () => { +test('rejected terminal schedules authority snapshot repair only', async () => { resetMocks() const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() const env = { APP_DB: {} } as unknown as Parameters< @@ -203,10 +197,5 @@ test('rejected terminal schedules delivery-event mirror only', async () => { await Promise.all(waitUntilPromises) expect(mocks.mirrorMailboxMessageGraphFromD1).not.toHaveBeenCalled() expect(mocks.processInboundDeliveryEffects).not.toHaveBeenCalled() - expect(mocks.mirrorMailboxDeliveryEventFromD1).toHaveBeenCalledWith({ - env, - db: env.APP_DB, - userId: 'user-ccc', - eventId: 'delivery-3', - }) + expect(mocks.authorityGet).toHaveBeenCalledWith('delivery-3') }) diff --git a/packages/worker/src/email/inbound-mailbox.ts b/packages/worker/src/email/inbound-mailbox.ts index 5076d63c1e..a78b834951 100644 --- a/packages/worker/src/email/inbound-mailbox.ts +++ b/packages/worker/src/email/inbound-mailbox.ts @@ -1,7 +1,7 @@ import { isSystemEmailOwner } from './email-owner.ts' +import { createUserInboundDeliveryAuthority } from './inbound-delivery-authority.ts' import { processInboundDeliveryEffects } from './inbound-effects.ts' import { - mirrorMailboxDeliveryEventFromD1, mirrorMailboxMessageGraphFromD1, type MailboxLiveMirrorEnv, } from './mailbox-live-mirror.ts' @@ -9,7 +9,8 @@ import { type EmailReportingEnv } from './reporting-events.ts' /** * Env surface for inbound Mailbox terminal coordination. - * D1 remains authoritative; Mailbox dual-write is best-effort after commit. + * Mailbox owns USER inbound delivery state; D1 is a synchronous compatibility + * snapshot used by graph fences and scheduled owner discovery. */ export type InboundMailboxEnv = Pick< Env, @@ -17,6 +18,7 @@ export type InboundMailboxEnv = Pick< | 'BUNDLE_ARTIFACTS_KV' | 'APP_BASE_URL' | 'USAGE_EVENTS' + | 'USER_METER' | 'MAILBOX' | 'EMAIL_EVENTS' > & @@ -44,8 +46,8 @@ export type InboundRejectedTerminalWorkInput = { /** * One ordered received-terminal task: * 1) bounded full message graph mirror (never throws) - * 2) D1 inbound delivery effects - * 3) on effects success only, mirror the updated delivery event + * 2) Mailbox-authoritative inbound delivery effects + * 3) final Mailbox → D1 delivery snapshot repair * * Attach to `ctx.waitUntil` when present; otherwise await. Catch/log once so * Mailbox/effects failures never throw into Email Routing. @@ -68,6 +70,7 @@ export async function scheduleInboundReceivedTerminalWork( db: input.env.APP_DB, userId: input.userId, messageId: input.messageId, + includeDeliveryEvents: false, }) await processInboundDeliveryEffects({ env: input.env, @@ -77,12 +80,11 @@ export async function scheduleInboundReceivedTerminalWork( durationMs: input.durationMs, waitUntil: nestedWaitUntil, }) - await mirrorMailboxDeliveryEventFromD1({ + const authority = createUserInboundDeliveryAuthority({ env: input.env, - db: input.env.APP_DB, userId: input.userId, - eventId: input.deliveryId, }) + await authority.get(input.deliveryId) })().catch((error: unknown) => { console.error(input.logLabel, error) }) @@ -95,7 +97,7 @@ export async function scheduleInboundReceivedTerminalWork( } /** - * Rejected-terminal delivery-event mirror only (no message graph). + * Rejected-terminal Mailbox → D1 snapshot repair only (no message graph). * `system:email` is skipped. Never throws into Email Routing. */ export async function scheduleInboundRejectedTerminalWork( @@ -103,12 +105,10 @@ export async function scheduleInboundRejectedTerminalWork( ) { if (isSystemEmailOwner(input.userId)) return - const task = mirrorMailboxDeliveryEventFromD1({ + const task = createUserInboundDeliveryAuthority({ env: input.env, - db: input.env.APP_DB, userId: input.userId, - eventId: input.deliveryId, - }) + }).get(input.deliveryId) if (input.ctx) { input.ctx.waitUntil(task) diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index adb4147f6d..e1300281f6 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -29,7 +29,6 @@ import { claimInboundDeliveryWindow, claimInboundDeliveryStorage, chargeSystemInboundDeliveryOnce, - chargeUserInboundDeliveryOnce, getInboundDeliveryWindow, getInboundDelivery, markInboundDeliveryRejected, @@ -42,6 +41,14 @@ import { type InboundDelivery, userInboundQuotaDay, } from './inbound-delivery.ts' +import { + createUserInboundDeliveryAuthority, + type UserInboundDeliveryAuthorityEnv, +} from './inbound-delivery-authority.ts' +import { + pruneUserExpiredInboundDedupePointers, + reconcileUserStaleInboundDeliveries, +} from './inbound-delivery-reconciliation-authority.ts' import { getPlatformEmailDomain, getSystemEmailDomain, @@ -85,12 +92,17 @@ async function rejectClaimedInboundDelivery(input: { message: ForwardableEmailMessage delivery: InboundDelivery reason: string + authority?: ReturnType }) { - const transitioned = await markInboundDeliveryRejected({ - db: input.db, - delivery: input.delivery, - reason: input.reason, - }).catch((error: unknown) => { + const transitioned = await ( + input.authority + ? input.authority.reject(input.delivery, input.reason) + : markInboundDeliveryRejected({ + db: input.db, + delivery: input.delivery, + reason: input.reason, + }) + ).catch((error: unknown) => { warnRejectionAuditWriteFailed(error) throw error }) @@ -120,6 +132,7 @@ async function parseAndStoreInboundEmail(input: { blobs: R2Bucket delivery: InboundDelivery parsed: Awaited> + authority?: ReturnType }) { const now = new Date().toISOString() try { @@ -130,6 +143,7 @@ async function parseAndStoreInboundEmail(input: { parsed: input.parsed, subjectNormalized: normalizeSubject(input.parsed.subject), now, + authority: input.authority, }) } catch (error) { if (error instanceof RetryableInboundStorageError) throw error @@ -141,18 +155,26 @@ async function parseAndStoreInboundEmail(input: { } async function cleanupInboundDurability(input: { - db: D1Database - blobs: R2Bucket + env: UserInboundDeliveryAuthorityEnv & { EMAIL_BLOBS: R2Bucket } userId: string }) { try { - await reconcileStaleInboundDeliveries(input) - await pruneExpiredInboundDedupePointers({ - db: input.db, - userId: input.userId, - }) + if (input.userId === systemEmailOwnerId) { + await reconcileStaleInboundDeliveries({ + db: input.env.APP_DB, + blobs: input.env.EMAIL_BLOBS, + userId: input.userId, + }) + await pruneExpiredInboundDedupePointers({ + db: input.env.APP_DB, + userId: input.userId, + }) + } else { + await reconcileUserStaleInboundDeliveries(input) + await pruneUserExpiredInboundDedupePointers(input) + } await deleteEmptyEmailThreads({ - db: input.db, + db: input.env.APP_DB, userId: input.userId, before: new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString(), limit: 20, @@ -276,6 +298,7 @@ export async function handleInboundEmail( db: env.APP_DB, stableUserId: userId, async write() { + const authority = createUserInboundDeliveryAuthority({ env, userId }) // Require email + canonical stable id together (same contract as // getUserPlan / isAccountEmailVerified) so a mismatched identity pair // cannot apply another account's plan or verification state. @@ -435,8 +458,7 @@ export async function handleInboundEmail( } await cleanupInboundDurability({ - db: env.APP_DB, - blobs: env.EMAIL_BLOBS, + env, userId, }) let rawMime: string @@ -458,18 +480,12 @@ export async function handleInboundEmail( quotaDay: userInboundQuotaDay(quotaNow), now: quotaNow, }) - const activeWindow = await getInboundDeliveryWindow({ - db: env.APP_DB, - userId, - fingerprint: candidateDelivery.fingerprint, - now: quotaNow, - }) - let delivery = activeWindow ?? candidateDelivery - let existingDelivery = await getInboundDelivery({ - db: env.APP_DB, - userId, - deliveryId: delivery.deliveryId, - }) + const activeWindow = await authority.getWindow( + candidateDelivery.fingerprint, + quotaNow, + ) + const delivery = activeWindow ?? candidateDelivery + let existingDelivery = await authority.get(delivery.deliveryId) if (!existingDelivery) { try { // New deliveries check storage bytes and stored-message caps @@ -534,26 +550,6 @@ export async function handleInboundEmail( return } } - if (!activeWindow) { - delivery = await claimInboundDeliveryWindow({ - db: env.APP_DB, - delivery: candidateDelivery, - now: quotaNow, - }) - if ( - !existingDelivery || - existingDelivery.deliveryId !== delivery.deliveryId - ) { - existingDelivery = await getInboundDelivery({ - db: env.APP_DB, - userId, - deliveryId: delivery.deliveryId, - }) - } - } - const waitUntil = ctx - ? (promise: Promise) => ctx.waitUntil(promise) - : undefined let claimedDelivery: InboundDelivery let chargedReceive = false try { @@ -564,9 +560,7 @@ export async function handleInboundEmail( ...delivery, quotaDay: userInboundQuotaDay(quotaNow), } - claimedDelivery = await chargeUserInboundDeliveryOnce({ - db: env.APP_DB, - env, + claimedDelivery = await authority.charge({ delivery: chargeCandidate, plan: account.plan, limit: resolveEmailResourceLimit( @@ -574,7 +568,6 @@ export async function handleInboundEmail( 'email_receives_per_day', ), now: quotaNow, - waitUntil, }) // The charge helper returns the candidate object only when this // invocation won the atomic D1 charge. A concurrently committed @@ -647,6 +640,7 @@ export async function handleInboundEmail( message, delivery: claimedDelivery, reason, + authority, }) if (!rejected) return await recordReceiveUsage({ outcome: 'error' }) @@ -658,12 +652,11 @@ export async function handleInboundEmail( }) return } - const storageClaim = await claimInboundDeliveryStorage({ - db: env.APP_DB, - delivery: claimedDelivery, - expectedAttachmentCount: parsed.attachments.length, - usageStartedAt: new Date(receiveStartedAtMs).toISOString(), - }) + const storageClaim = await authority.claimStorage( + claimedDelivery, + parsed.attachments.length, + new Date(receiveStartedAtMs).toISOString(), + ) if (!storageClaim.claimed) { if (storageClaim.delivery?.state === 'received') { await scheduleInboundReceivedTerminalWork({ @@ -689,18 +682,18 @@ export async function handleInboundEmail( blobs: env.EMAIL_BLOBS, delivery: storageClaim.delivery, parsed, + authority, }) } catch (error) { - await releaseInboundDeliveryStorage({ - db: env.APP_DB, - delivery: storageClaim.delivery, - }).catch((releaseError) => { - console.error( - 'inbound-email-storage-lease-release-failed', - storageClaim.delivery.deliveryId, - releaseError, - ) - }) + await authority + .releaseStorage(storageClaim.delivery) + .catch((releaseError) => { + console.error( + 'inbound-email-storage-lease-release-failed', + storageClaim.delivery.deliveryId, + releaseError, + ) + }) throw error } // Mailbox dual-write only after durable D1/R2 commit + finalization win. @@ -729,6 +722,8 @@ async function handleSystemInboundEmail(input: { | 'BUNDLE_ARTIFACTS_KV' | 'APP_BASE_URL' | 'USAGE_EVENTS' + | 'MAILBOX' + | 'USER_METER' > recipient: string localPart: SystemEmailLocal @@ -798,8 +793,7 @@ async function handleSystemInboundEmail(input: { } await cleanupInboundDurability({ - db: input.env.APP_DB, - blobs: input.env.EMAIL_BLOBS, + env: input.env, userId: systemEmailOwnerId, }) let rawMime: string diff --git a/packages/worker/src/email/inbound.workers.test.ts b/packages/worker/src/email/inbound.workers.test.ts index e00a53a583..7cd5013b4b 100644 --- a/packages/worker/src/email/inbound.workers.test.ts +++ b/packages/worker/src/email/inbound.workers.test.ts @@ -4,10 +4,13 @@ import { utcDayKey } from '@kody-internal/shared/date-keys.ts' import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' import { handleInboundEmail } from './inbound.ts' import { processInboundDeliveryEffects } from './inbound-effects.ts' +import { createUserInboundDeliveryAuthority } from './inbound-delivery-authority.ts' +import { mailboxRpc } from './mailbox-client.ts' import { buildInboundDelivery, claimInboundDeliveryWindow, claimInboundDeliveryStorage, + getInboundDelivery, inboundDeliveryDedupeWindowMs, InboundDeliveryLeaseLostError, markInboundDeliveryRejected, @@ -1475,9 +1478,8 @@ test('identical MIME from distinct envelope senders creates separate deliveries' expect(await readUserDailyReceiveCount(userId)).toBe(2) }) -test('inbound post-commit bookkeeping failure keeps one stored row without refund or retry throw', async () => { +test('D1 fence mirror failure fails closed and replay repairs without a second charge', async () => { silenceIncidentalRuntimeWarnings() - silenceExpectedConsoleErrors(['inbound-email-post-commit-bookkeeping-failed']) await ensureEmailTestSchema(env.APP_DB) const username = `postcommit-${crypto.randomUUID().slice(0, 8)}` const accountEmail = `postcommit-${crypto.randomUUID()}@example.com` @@ -1488,24 +1490,27 @@ test('inbound post-commit bookkeeping failure keeps one stored row without refun email: accountEmail, username, }) + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Post-commit bookkeeping', + 'Message-ID: ', + '', + 'Body', + ].join('\r\n') const message = createForwardableEmailMessage({ from: 'sender@example.net', to: address, - raw: [ - 'From: Sender ', - `To: ${address}`, - 'Subject: Post-commit bookkeeping', - 'Message-ID: ', - '', - 'Body', - ].join('\r\n'), + raw, }) const failingEnv = { ...createInboundEnv(), APP_DB: createPostCommitBookkeepingFailureDb(), } as Parameters[1] - await handleInboundEmail(message, failingEnv) + await expect(handleInboundEmail(message, failingEnv)).rejects.toBeInstanceOf( + RetryableInboundStorageError, + ) expect(message.rejectedReason).toBeNull() expect(await readUserDailyReceiveCount(userId)).toBe(1) expect( @@ -1515,9 +1520,18 @@ test('inbound post-commit bookkeeping failure keeps one stored row without refun limit: 10, }), ).toHaveLength(1) + await handleInboundEmail( + createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }), + createInboundEnv(), + ) + expect(await readUserDailyReceiveCount(userId)).toBe(1) }) -test('delivery-ledger finalization failure retries before usage or subscription success', async () => { +test('Mailbox finalization runs effects once and ignores D1 authority reset attempts', async () => { silenceIncidentalRuntimeWarnings() await ensureEmailTestSchema(env.APP_DB) await ensureUsageRollupsTestSchema(env.APP_DB) @@ -1530,33 +1544,6 @@ test('delivery-ledger finalization failure retries before usage or subscription email: accountEmail, username, }) - let finalizationFailed = false - const failingDb = new Proxy(env.APP_DB, { - get(target, property, receiver) { - if (property === 'prepare') { - return (query: string) => { - const statement = target.prepare(query) - if ( - finalizationFailed || - !query.includes('UPDATE email_delivery_events') || - !query.includes('SET message_id = ?') - ) { - return statement - } - return { - bind: () => ({ - run: async () => { - finalizationFailed = true - throw new Error('simulated ledger finalization failure') - }, - }), - } - } - } - const value = Reflect.get(target, property, receiver) - return typeof value === 'function' ? value.bind(target) : value - }, - }) as D1Database const raw = [ 'From: Sender ', `To: ${address}`, @@ -1571,20 +1558,15 @@ test('delivery-ledger finalization failure retries before usage or subscription raw, }) - await expect( - handleInboundEmail(first, { - ...createInboundEnv(), - APP_DB: failingDb, - }), - ).rejects.toBeInstanceOf(RetryableInboundStorageError) + await handleInboundEmail(first, createInboundEnv()) expect( await env.APP_DB.prepare( `SELECT event_count FROM usage_rollups WHERE user_id = ? AND metric = 'email_received'`, ) .bind(userId) - .first(), - ).toBeNull() + .first<{ event_count: number }>(), + ).toEqual({ event_count: 1 }) const retry = createForwardableEmailMessage({ from: 'sender@example.net', @@ -1641,11 +1623,13 @@ test('delivery-ledger finalization failure retries before usage or subscription ) .bind(delivery.id, userId) .run() - await processInboundDeliveryEffects({ - env: createInboundEnv(), - userId, - deliveryId: delivery.id, - }) + expect( + await processInboundDeliveryEffects({ + env: createInboundEnv(), + userId, + deliveryId: delivery.id, + }), + ).toEqual({ outcome: 'usage-only' }) expect( await env.APP_DB.prepare( `SELECT event_count, total_duration_ms FROM usage_rollups @@ -1653,7 +1637,7 @@ test('delivery-ledger finalization failure retries before usage or subscription ) .bind(userId) .first<{ event_count: number; total_duration_ms: number }>(), - ).toEqual({ event_count: 1, total_duration_ms: 4321 }) + ).toBeNull() }) test('production usage outbox records one durable D1 event without retry data points', async () => { @@ -2044,12 +2028,58 @@ test('stale rejection cannot overwrite finalized delivery usage', async () => { usageMonth: '2026-07', usageBytes: 456, }) - await processInboundDeliveryEffects({ + expect( + await getInboundDelivery({ + db: env.APP_DB, + userId, + deliveryId: pending.deliveryId, + }), + ).toMatchObject({ state: 'received', finalizationToken: expect.any(String) }) + const authorityDelivery = await createUserInboundDeliveryAuthority({ + env: createInboundEnv(), + userId, + }).get(pending.deliveryId) + const mailboxEvents = await mailboxRpc({ + env: createInboundEnv(), + userId, + }).listDeliveryEvents({ limit: 10 }) + expect(mailboxEvents[0]).toMatchObject({ + id: pending.deliveryId, + provider: 'cloudflare-email-routing', + state: 'received', + }) + expect(JSON.parse(mailboxEvents[0]!.detailJson)).toMatchObject({ + fingerprint: expect.any(String), + deliveryId: pending.deliveryId, + messageId: pending.messageId, + threadId: pending.threadId, + rawMimeKey: pending.rawMimeKey, + inboxId: pending.inboxId, + recipient: pending.recipient, + envelopeFrom: pending.envelopeFrom, + quotaDay: pending.quotaDay, + dedupeExpiresAt: pending.dedupeExpiresAt, + }) + expect( + await mailboxRpc({ + env: createInboundEnv(), + userId, + }).getInboundDelivery({ + ownerId: userId, + deliveryId: pending.deliveryId, + }), + ).not.toBeNull() + expect(authorityDelivery).toMatchObject({ + state: 'received', + finalizationToken: expect.any(String), + }) + const effectResult = await processInboundDeliveryEffects({ env: createInboundEnv(), userId, deliveryId: pending.deliveryId, now, }) + expect(effectResult).toEqual({ outcome: 'complete' }) expect( await markInboundDeliveryRejected({ diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 0bea8aec63..1fd38bffb9 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -21,6 +21,11 @@ import { touchMailboxThread, updateMailboxMessageDelivery, } from './mailbox-mutations.ts' +import { + claimMailboxInboundDeliveryCleanup, + markMailboxInboundDeliveryOrphanCleaned, + releaseMailboxInboundDeliveryCleanup, +} from './mailbox-inbound-cleanup-ledger.ts' import { claimMailboxInboundDeliveryStorage, claimMailboxInboundDeliveryWindow, @@ -84,23 +89,14 @@ import { * external attachment bytes stay in `EMAIL_BLOBS`; rows retain keys. * `system:email` stays in D1 by design. * - * Dual-write / read-cutover live paths are wired elsewhere. Additive step 2a - * inbound ledger CAS RPCs are exposed but not live-wired (D1 remains - * authority for inbound ledger/effects). + * Dual-write / read-cutover live paths are wired elsewhere. USER inbound + * ledger/effect transitions are authoritative here; `system:email` remains D1. */ class MailboxBase extends DurableObject implements MailboxRpc { private readonly store: MailboxStore - /** - * In-isolate cache: once an alarm is scheduled, hot writes skip - * getAlarm/setAlarm. Cleared when retention becomes idle or a write may - * need an earlier wake. - */ + /** In-isolate alarm scheduling caches. */ private retentionAlarmArmed = false - /** - * In-isolate: no future retention work. Cleared on every data write so - * ensureRetentionAlarm re-arms for the new row's due-time. - */ private retentionIdleConfirmed = false constructor(ctx: DurableObjectState, env: Env) { @@ -507,11 +503,6 @@ class MailboxBase extends DurableObject implements MailboxRpc { return this.store.listBlobReferences(input) } - /** - * Additive step 2a inbound ledger CAS RPCs. Not live-wired — D1 remains - * authority. Mirror `upsertDeliveryEvent(s)` stay the compatibility path. - */ - async getInboundDelivery(input: { ownerId: string deliveryId: string @@ -688,6 +679,55 @@ class MailboxBase extends DurableObject implements MailboxRpc { return result } + async claimInboundDeliveryCleanup(input: { + ownerId: string + deliveryId: string + now?: string + }) { + let result!: Awaited> + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = claimMailboxInboundDeliveryCleanup(this.ctx.storage.sql, input) + }) + return result + } + + async releaseInboundDeliveryCleanup(input: { + ownerId: string + deliveryId: string + cleanupLease: string + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = releaseMailboxInboundDeliveryCleanup(this.ctx.storage.sql, input) + }) + return result + } + + async markInboundDeliveryOrphanCleaned(input: { + ownerId: string + deliveryId: string + cleanupLease: string + outcome: 'deleted' | 'delete-failed' + now?: string + }) { + let result!: Awaited< + ReturnType + > + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = markMailboxInboundDeliveryOrphanCleaned( + this.ctx.storage.sql, + input, + ) + }) + return result + } + async claimInboundUsageEffect(input: { ownerId: string deliveryId: string diff --git a/packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts b/packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts new file mode 100644 index 0000000000..dd172c1e07 --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts @@ -0,0 +1,196 @@ +import { assertMailboxNonEmptyString } from './mailbox-types.ts' +import { + detailJsonFromMailboxInboundSnapshot, + mailboxInboundProvider, + mailboxInboundReconciliationRetryMs, + mailboxInboundStorageLeaseMs, + normalizeMailboxInboundNow, + readMailboxInboundDeliveryById, + type MailboxClaimInboundDeliveryCleanupResult, + type MailboxInboundDeliverySnapshot, + type MailboxMarkInboundDeliveryOrphanCleanedResult, + type MailboxReleaseInboundDeliveryCleanupResult, +} from './mailbox-inbound-ledger.ts' +import { mailboxInboundOrphanVerificationMs } from './mailbox-inbound-ledger-shared.ts' + +export function claimMailboxInboundDeliveryCleanup( + sql: SqlStorage, + input: { deliveryId: string; now?: string }, +): MailboxClaimInboundDeliveryCleanupResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if (!current) return { status: 'not-claimed', delivery: null } + const leaseExpiredBefore = new Date( + Date.parse(now) - mailboxInboundStorageLeaseMs, + ).toISOString() + const claimable = + current.state === 'pending' || + (current.state === 'storing' && + current.storageLeaseAt != null && + current.storageLeaseAt < leaseExpiredBefore) || + (current.state === 'cleaning' && + current.cleanupLeaseAt != null && + current.cleanupLeaseAt < leaseExpiredBefore) || + (current.state === 'orphan-cleaned' && + current.cleanupRetryAt != null && + current.cleanupRetryAt <= now) + if (!claimable) return { status: 'not-claimed', delivery: current } + + const cleanupLease = crypto.randomUUID() + const next: MailboxInboundDeliverySnapshot = { + ...current, + state: 'cleaning', + cleanupLease, + cleanupLeaseAt: now, + updatedAt: now, + } + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + state = 'cleaning', + cleanup_lease = ?, + cleanup_lease_at = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND ( + state = 'pending' + OR (state = 'storing' AND storage_lease_at < ?) + OR (state = 'cleaning' AND cleanup_lease_at < ?) + OR (state = 'orphan-cleaned' AND cleanup_retry_at <= ?) + )`, + detailJsonFromMailboxInboundSnapshot(next), + cleanupLease, + now, + now, + deliveryId, + mailboxInboundProvider, + leaseExpiredBefore, + leaseExpiredBefore, + now, + ) + const after = readMailboxInboundDeliveryById(sql, deliveryId) + if ( + cursor.rowsWritten > 0 && + after?.state === 'cleaning' && + after.cleanupLease === cleanupLease + ) { + return { status: 'claimed', delivery: after } + } + return { status: 'not-claimed', delivery: after } +} + +export function releaseMailboxInboundDeliveryCleanup( + sql: SqlStorage, + input: { + deliveryId: string + cleanupLease: string + now?: string + }, +): MailboxReleaseInboundDeliveryCleanupResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const cleanupLease = assertMailboxNonEmptyString( + input.cleanupLease, + 'cleanupLease', + ) + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if ( + !current || + current.state !== 'cleaning' || + current.cleanupLease !== cleanupLease + ) { + return { status: 'not-held' } + } + const next: MailboxInboundDeliverySnapshot = { + ...current, + state: 'pending', + updatedAt: now, + } + delete next.cleanupLease + delete next.cleanupLeaseAt + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + state = 'pending', + cleanup_lease = NULL, + cleanup_lease_at = NULL, + updated_at = ? + WHERE id = ? + AND provider = ? + AND state = 'cleaning' + AND cleanup_lease = ?`, + detailJsonFromMailboxInboundSnapshot(next), + now, + deliveryId, + mailboxInboundProvider, + cleanupLease, + ) + if (cursor.rowsWritten < 1) return { status: 'not-held' } + const after = readMailboxInboundDeliveryById(sql, deliveryId) + return after + ? { status: 'released', delivery: after } + : { status: 'not-held' } +} + +export function markMailboxInboundDeliveryOrphanCleaned( + sql: SqlStorage, + input: { + deliveryId: string + cleanupLease: string + outcome: 'deleted' | 'delete-failed' + now?: string + }, +): MailboxMarkInboundDeliveryOrphanCleanedResult { + const now = normalizeMailboxInboundNow(input.now) + const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const cleanupLease = assertMailboxNonEmptyString( + input.cleanupLease, + 'cleanupLease', + ) + const current = readMailboxInboundDeliveryById(sql, deliveryId) + if ( + !current || + current.state !== 'cleaning' || + current.cleanupLease !== cleanupLease + ) { + return { status: 'lease-lost' } + } + const retryDelayMs = + input.outcome === 'deleted' + ? mailboxInboundOrphanVerificationMs + : mailboxInboundReconciliationRetryMs + const next: MailboxInboundDeliverySnapshot = { + ...current, + state: 'orphan-cleaned', + cleanupRetryAt: new Date(Date.parse(now) + retryDelayMs).toISOString(), + updatedAt: now, + } + delete next.cleanupLease + delete next.cleanupLeaseAt + const cursor = sql.exec( + `UPDATE email_delivery_events + SET detail_json = ?, + state = 'orphan-cleaned', + cleanup_lease = NULL, + cleanup_lease_at = NULL, + cleanup_retry_at = ?, + updated_at = ? + WHERE id = ? + AND provider = ? + AND state = 'cleaning' + AND cleanup_lease = ?`, + detailJsonFromMailboxInboundSnapshot(next), + next.cleanupRetryAt, + now, + deliveryId, + mailboxInboundProvider, + cleanupLease, + ) + if (cursor.rowsWritten < 1) return { status: 'lease-lost' } + const after = readMailboxInboundDeliveryById(sql, deliveryId) + return after + ? { status: 'orphan-cleaned', delivery: after } + : { status: 'lease-lost' } +} diff --git a/packages/worker/src/email/mailbox-inbound-effect-ledger.ts b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts index 913940b231..5f582cf63e 100644 --- a/packages/worker/src/email/mailbox-inbound-effect-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-effect-ledger.ts @@ -1,5 +1,5 @@ /** - * Additive Mailbox USER inbound effect CAS primitives (step 2a). + * Mailbox USER inbound effect authority CAS primitives. * * Lease claim / complete / fail for usage and subscription effects. Does not * perform external usage recording or subscription dispatch — callers do that diff --git a/packages/worker/src/email/mailbox-inbound-ledger-shared.ts b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts index 3a87b0de59..85ce3e455b 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger-shared.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger-shared.ts @@ -1,6 +1,5 @@ /** - * Shared snapshot / detail helpers and constants for Mailbox inbound ledger - * CAS (step 2a). + * Shared snapshot / detail helpers and constants for Mailbox inbound authority. */ import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' @@ -16,6 +15,7 @@ export const mailboxInboundProvider = 'cloudflare-email-routing' export const mailboxInboundDedupeProvider = 'cloudflare-email-routing-dedupe' export const mailboxInboundStorageLeaseMs = 5 * 60 * 1000 export const mailboxInboundReconciliationRetryMs = 15 * 60 * 1000 +export const mailboxInboundOrphanVerificationMs = 60 * 60 * 1000 export const mailboxInboundDeliveryDedupeWindowMs = 48 * 60 * 60 * 1000 export const mailboxStaleInboundDeliveryAgeMs = 48 * 60 * 60 * 1000 export const mailboxUsageEffectLeaseMs = 5 * 60 * 1000 @@ -112,7 +112,10 @@ export function snapshotFromMailboxDeliveryEventRow( const inboxId = optionalDetailString(detail['inboxId']) ?? row.inboxId ?? undefined const recipient = optionalDetailString(detail['recipient']) - const envelopeFrom = optionalDetailString(detail['envelopeFrom']) + const envelopeFrom = + typeof detail['envelopeFrom'] === 'string' + ? detail['envelopeFrom'] + : undefined const quotaDay = optionalDetailString(detail['quotaDay']) const dedupeExpiresAt = row.dedupeExpiresAt ?? optionalDetailString(detail['dedupeExpiresAt']) @@ -123,7 +126,7 @@ export function snapshotFromMailboxDeliveryEventRow( !rawMimeKey || !inboxId || !recipient || - !envelopeFrom || + envelopeFrom === undefined || !quotaDay || !dedupeExpiresAt ) { diff --git a/packages/worker/src/email/mailbox-inbound-ledger.ts b/packages/worker/src/email/mailbox-inbound-ledger.ts index c607316ea3..47e7c8d277 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.ts @@ -1,11 +1,11 @@ /** - * Additive Mailbox USER inbound ledger CAS primitives (step 2a). + * Mailbox USER inbound ledger authority CAS primitives. * * Owner-bound SQLite helpers matching D1 USER transitions in * `inbound-delivery.ts`. Effect lease CAS lives in * `mailbox-inbound-effect-ledger.ts`. No external usage/subscription side - * effects. Mirror upsert RPCs remain the compatibility write path; these CAS - * RPCs are not live-wired (D1 stays authority). `system:email` stays on D1. + * effects. D1 receives synchronous compatibility snapshots after CAS; + * `system:email` stays on D1. */ import { emailRawMimeKey } from './blob-keys.ts' @@ -105,6 +105,18 @@ export type MailboxDeferInboundDeliveryReconcileResult = | { status: 'missing' } | { status: 'not-applicable' } +export type MailboxClaimInboundDeliveryCleanupResult = + | { status: 'claimed'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'not-claimed'; delivery: MailboxInboundDeliverySnapshot | null } + +export type MailboxReleaseInboundDeliveryCleanupResult = + | { status: 'released'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'not-held' } + +export type MailboxMarkInboundDeliveryOrphanCleanedResult = + | { status: 'orphan-cleaned'; delivery: MailboxInboundDeliverySnapshot } + | { status: 'lease-lost' } + export type MailboxPruneExpiredInboundDedupeResult = { pruned: number } export type MailboxListDueStaleInboundDeliveriesResult = { @@ -171,6 +183,24 @@ export type MailboxInboundDeliveryLedgerRpc = { deliveryId: string now?: string }) => Promise + claimInboundDeliveryCleanup: (input: { + ownerId: string + deliveryId: string + now?: string + }) => Promise + releaseInboundDeliveryCleanup: (input: { + ownerId: string + deliveryId: string + cleanupLease: string + now?: string + }) => Promise + markInboundDeliveryOrphanCleaned: (input: { + ownerId: string + deliveryId: string + cleanupLease: string + outcome: 'deleted' | 'delete-failed' + now?: string + }) => Promise listDueStaleInboundDeliveries: (input: { ownerId: string now?: string @@ -225,7 +255,9 @@ function assertInsertInput( ) assertMailboxNonEmptyString(delivery.inboxId, 'delivery.inboxId') assertMailboxNonEmptyString(delivery.recipient, 'delivery.recipient') - assertMailboxNonEmptyString(delivery.envelopeFrom, 'delivery.envelopeFrom') + if (typeof delivery.envelopeFrom !== 'string') { + throw new Error('Mailbox delivery.envelopeFrom must be a string.') + } assertMailboxNonEmptyString(delivery.quotaDay, 'delivery.quotaDay') return { deliveryId, messageId, fingerprint, threadId, rawMimeKey } } @@ -322,11 +354,11 @@ export function claimMailboxInboundDeliveryWindow( fingerprint = excluded.fingerprint, dedupe_expires_at = excluded.dedupe_expires_at, usage_started_at = excluded.usage_started_at, - provider = excluded.provider, provider_event_id = excluded.provider_event_id, created_at = excluded.created_at, updated_at = excluded.updated_at - WHERE email_delivery_events.dedupe_expires_at <= ?`, + WHERE email_delivery_events.provider = ? + AND email_delivery_events.dedupe_expires_at <= ?`, pointerId, input.delivery.inboxId, mailboxInboundDedupeProvider, @@ -337,9 +369,13 @@ export function claimMailboxInboundDeliveryWindow( input.delivery.usageStartedAt ?? null, now, now, + mailboxInboundDedupeProvider, now, ) const row = readMailboxDeliveryEventRow(sql, pointerId) + if (row?.provider !== mailboxInboundDedupeProvider) { + throw new Error('Mailbox inbound dedupe pointer failed its provider fence.') + } const snapshot = row ? snapshotFromMailboxDeliveryEventRow(row) : null if (!snapshot) { throw new Error('Failed to resolve the inbound delivery dedupe window.') diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts index ebc255abcd..bf94d26d8b 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -21,7 +21,7 @@ import { uniqueUserId, } from './mailbox-test-helpers.ts' -test('Mailbox inbound ledger CAS covers USER transition matrix without live flip', async () => { +test('Mailbox inbound ledger CAS covers USER authority transition matrix', async () => { silenceIncidentalRuntimeWarnings() const ownerA = uniqueUserId('ledger-a') const ownerB = uniqueUserId('ledger-b') @@ -252,6 +252,25 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip expect( dueStale.deliveries.some((d) => d.deliveryId === stalePending.deliveryId), ).toBe(true) + const cleanupClaim = await mailboxA.claimInboundDeliveryCleanup({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + now, + }) + expect(cleanupClaim.status).toBe('claimed') + if ( + cleanupClaim.status !== 'claimed' || + !cleanupClaim.delivery.cleanupLease + ) { + throw new Error('expected cleanup claim') + } + const cleanupRelease = await mailboxA.releaseInboundDeliveryCleanup({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + cleanupLease: cleanupClaim.delivery.cleanupLease, + now, + }) + expect(cleanupRelease.status).toBe('released') const foreign = insertInput(ownerB, { fingerprint: 'fp-b', @@ -305,6 +324,31 @@ test('Mailbox inbound ledger CAS covers USER transition matrix without live flip now, }) expect(deferred.status).toBe('deferred') + const orphanClaim = await mailboxA.claimInboundDeliveryCleanup({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + now, + }) + if (orphanClaim.status !== 'claimed' || !orphanClaim.delivery.cleanupLease) { + throw new Error('expected orphan cleanup claim') + } + expect( + await mailboxA.markInboundDeliveryOrphanCleaned({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + cleanupLease: 'stale-cleanup-lease', + outcome: 'deleted', + now, + }), + ).toEqual({ status: 'lease-lost' }) + const orphaned = await mailboxA.markInboundDeliveryOrphanCleaned({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + cleanupLease: orphanClaim.delivery.cleanupLease, + outcome: 'delete-failed', + now, + }) + expect(orphaned.status).toBe('orphan-cleaned') // Mirror upsert compatibility still works alongside ledger rows. const mirror = await mailboxA.upsertDeliveryEvent({ diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts index 493a6c0e66..e0252baf4c 100644 --- a/packages/worker/src/email/mailbox-live-mirror.ts +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -21,9 +21,11 @@ import { type EmailThreadRecord } from './types.ts' /** * High-level D1 → Mailbox live-mirror orchestration. * - * D1 remains authoritative. These helpers load cohesive snapshots from D1 and - * call best-effort mirror RPCs; they never throw into caller paths and do not - * perform deletes or inbound-lifecycle special cases. + * D1 remains authoritative for the message graph and non-USER-inbound events. + * These helpers load cohesive snapshots from D1 and call best-effort mirror + * RPCs; they never throw into caller paths and do not perform deletes or + * inbound-lifecycle special cases. USER inbound terminal work explicitly omits + * delivery events because those snapshots flow Mailbox → D1. * * Callers that just created a thread may pass it via `thread` to avoid a * round-trip; otherwise the graph helper loads it with `getEmailThreadById`. @@ -132,6 +134,8 @@ export async function mirrorMailboxMessageGraphFromD1(input: { userId: string messageId: string thread?: EmailThreadRecord | null + /** USER inbound terminal work keeps its DO-authoritative event untouched. */ + includeDeliveryEvents?: boolean }): Promise { try { const message = await getEmailMessageById({ @@ -168,6 +172,9 @@ export async function mirrorMailboxMessageGraphFromD1(input: { message, attachments, }) + if (input.includeDeliveryEvents === false) { + return emptyGraphSummary(input.messageId, messageResult) + } // Newest max+1 from D1 (chrono-restored). When truncated, keep the // trailing newest max — drop the oldest overflow row at index 0. diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 4a8a7f1cf8..98d7ee9498 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -479,9 +479,8 @@ export type MailboxUpsertDeliveryEventsResult = { } /** - * Mirror / read / retention / purge surface. Inbound ledger CAS RPCs - * (additive step 2a) are intersected below — not live-wired; D1 remains - * authority. + * Mirror / read / retention / purge surface. Authoritative USER inbound ledger + * CAS RPCs are intersected below; `system:email` remains D1-only. */ type MailboxCoreRpc = { mirrorMessage: (input: { @@ -500,8 +499,8 @@ type MailboxCoreRpc = { attachments?: Array }) => Promise<{ ok: true; accepted: boolean }> /** - * Compatibility mirror upsert for complete delivery-event snapshots. - * Remains available during step 2a; CAS ledger RPCs are additive. + * Complete delivery-event snapshot upsert. USER inbound only uses this for + * the one-time pre-deploy D1 bootstrap bridge. */ upsertDeliveryEvent: (input: { ownerId: string diff --git a/packages/worker/src/email/reconcile-inbound-deliveries.ts b/packages/worker/src/email/reconcile-inbound-deliveries.ts index fdbb018a31..02558260c3 100644 --- a/packages/worker/src/email/reconcile-inbound-deliveries.ts +++ b/packages/worker/src/email/reconcile-inbound-deliveries.ts @@ -4,6 +4,10 @@ import { staleInboundDeliveryAgeMs, } from './inbound-delivery.ts' import { reconcileInboundDeliveryEffectsForUser } from './inbound-effects.ts' +import { + pruneUserExpiredInboundDedupePointers, + reconcileUserStaleInboundDeliveries, +} from './inbound-delivery-reconciliation-authority.ts' import { systemEmailOwnerId } from './system-email.ts' import { withAccountWriteLease } from '#worker/account/deletion-state.ts' @@ -19,6 +23,8 @@ export async function sweepStaleInboundDeliveries(input: { | 'BUNDLE_ARTIFACTS_KV' | 'APP_BASE_URL' | 'USAGE_EVENTS' + | 'MAILBOX' + | 'USER_METER' > now?: Date }) { @@ -126,18 +132,32 @@ export async function sweepStaleInboundDeliveries(input: { if (Date.now() >= deadlineMs) break try { const reconcileUser = async () => { - const result = await reconcileStaleInboundDeliveries({ - db: input.env.APP_DB, - blobs: input.env.EMAIL_BLOBS, - userId, - now, - deadlineMs, - }) - const pruned = await pruneExpiredInboundDedupePointers({ - db: input.env.APP_DB, - userId, - now, - }) + const systemOwner = userId === systemEmailOwnerId + const result = systemOwner + ? await reconcileStaleInboundDeliveries({ + db: input.env.APP_DB, + blobs: input.env.EMAIL_BLOBS, + userId, + now, + deadlineMs, + }) + : await reconcileUserStaleInboundDeliveries({ + env: input.env, + userId, + now, + deadlineMs, + }) + const pruned = systemOwner + ? await pruneExpiredInboundDedupePointers({ + db: input.env.APP_DB, + userId, + now, + }) + : await pruneUserExpiredInboundDedupePointers({ + env: input.env, + userId, + now, + }) const effectResult = await reconcileInboundDeliveryEffectsForUser({ env: input.env, userId, diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index 19fea67aa7..cff2eab5c5 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -353,6 +353,15 @@ export async function storeIdempotentInboundEmail(input: { parsed: ParsedInboundEmail subjectNormalized: string now: string + authority?: { + get: (deliveryId: string) => Promise + receive: (input: { + delivery: InboundDelivery + usageDurationMs: number + usageMonth: string + usageBytes: number + }) => Promise + } }) { const { delivery, parsed } = input if (!delivery.storageLease) { @@ -496,21 +505,30 @@ export async function storeIdempotentInboundEmail(input: { let finalizedDelivery: InboundDelivery try { - finalizedDelivery = await markInboundDeliveryReceived({ - db: input.db, + const finalization = { delivery, usageDurationMs: delivery.usageStartedAt ? Date.now() - Date.parse(delivery.usageStartedAt) : 0, usageMonth: (stored.receivedAt ?? stored.createdAt).slice(0, 7), usageBytes: stored.rawSize ?? 0, - }) + } + finalizedDelivery = input.authority + ? await input.authority.receive(finalization) + : await markInboundDeliveryReceived({ + db: input.db, + ...finalization, + }) } catch (error) { - const committed = await getInboundDelivery({ - db: input.db, - userId: delivery.userId, - deliveryId: delivery.deliveryId, - }).catch(() => null) + const committed = await ( + input.authority + ? input.authority.get(delivery.deliveryId) + : getInboundDelivery({ + db: input.db, + userId: delivery.userId, + deliveryId: delivery.deliveryId, + }) + ).catch(() => null) if (committed?.state !== 'received') { throw new RetryableInboundStorageError( 'Failed to finalize the inbound delivery ledger; the stable delivery will be retried.', diff --git a/packages/worker/src/email/test-schema.ts b/packages/worker/src/email/test-schema.ts index 6d35927cf4..c3ff0beb15 100644 --- a/packages/worker/src/email/test-schema.ts +++ b/packages/worker/src/email/test-schema.ts @@ -8,6 +8,7 @@ export async function ensureEmailTestSchema(db: D1Database) { `DROP TABLE IF EXISTS system_email_daily_counters;`, `DROP TABLE IF EXISTS email_sender_policies;`, `DROP TABLE IF EXISTS email_sender_rules;`, + `DROP TABLE IF EXISTS email_inbound_usage_effects;`, `DROP TABLE IF EXISTS email_delivery_events;`, `DROP TABLE IF EXISTS email_attachments;`, `DROP TABLE IF EXISTS email_outbound_provider_index;`, @@ -154,13 +155,52 @@ ON email_sender_rules(user_id, kind, value);`, usage_month TEXT, usage_bytes INTEGER, usage_duration_ms INTEGER, - created_at TEXT NOT NULL + state TEXT CHECK ( + state IS NULL OR state IN ( + 'pending', 'storing', 'cleaning', 'received', 'rejected', 'orphan-cleaned' + ) + ), + fingerprint TEXT, + storage_lease TEXT, + storage_lease_at TEXT, + cleanup_lease TEXT, + cleanup_lease_at TEXT, + cleanup_retry_at TEXT, + expected_attachment_count INTEGER, + finalization_token TEXT, + reconcile_after TEXT, + dedupe_expires_at TEXT, + usage_effect_suppressed_at TEXT, + usage_started_at TEXT, + usage_effect_retry_at TEXT, + usage_effect_lease TEXT, + usage_effect_lease_at TEXT, + subscription_effect_state TEXT CHECK ( + subscription_effect_state IS NULL OR subscription_effect_state IN ( + 'pending', 'processing', 'complete', 'dead-letter' + ) + ), + subscription_effect_lease TEXT, + subscription_effect_lease_at TEXT, + subscription_effect_retry_at TEXT, + subscription_effect_attempt_count INTEGER, + subscription_effect_dead_letter_at TEXT, + subscription_effect_last_error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT );`, `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_pending_effects ON email_delivery_events(user_id, created_at) WHERE provider = 'cloudflare-email-routing' AND event_type = 'received' AND needs_effect_reconcile = 1;`, + `CREATE TABLE IF NOT EXISTS email_inbound_usage_effects ( + user_id TEXT NOT NULL, + delivery_id TEXT NOT NULL, + finalization_token TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (user_id, delivery_id, finalization_token) +);`, `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_recorded_usage_month ON email_delivery_events(usage_month, user_id) WHERE provider = 'cloudflare-email-routing' diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index 9cf767641c..d58462fc96 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -527,6 +527,10 @@ { "filename": "0128-email-outbound-provider-index.sql", "sha256": "b51c4f9cc4416cd81c1c1648311e0267e28a208d386d4729295b1d3dad54ed5a" + }, + { + "filename": "0129-email-inbound-mailbox-authority-mirror.sql", + "sha256": "41df09101819cf992d33504013ebf016d7bbbf49b310d08a02a71cb6b5b6dfe1" } ] } From ac46a01996223f040dc5d2106896c655163a54d8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 16:40:40 +0000 Subject: [PATCH 55/62] fix(email): harden Mailbox inbound authority Co-authored-by: Kent C. Dodds --- ...email-inbound-mailbox-authority-mirror.sql | 3 +- .../inbound-delivery-authority.node.test.ts | 271 +++++- .../src/email/inbound-delivery-authority.ts | 788 ++++++++---------- .../src/email/inbound-delivery-projection.ts | 329 ++++++++ packages/worker/src/email/inbound-effects.ts | 16 +- ...ox-authority-mirror-migration.node.test.ts | 262 ++++++ .../src/email/inbound-mailbox.node.test.ts | 23 + packages/worker/src/email/inbound-mailbox.ts | 6 +- packages/worker/src/email/inbound.ts | 5 +- .../worker/src/email/inbound.workers.test.ts | 14 + .../src/email/reconcile-inbound-deliveries.ts | 119 ++- packages/worker/src/email/service.ts | 11 +- packages/worker/src/email/test-schema.ts | 3 +- tools/migration-ledger.json | 2 +- 14 files changed, 1336 insertions(+), 516 deletions(-) create mode 100644 packages/worker/src/email/inbound-delivery-projection.ts create mode 100644 packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts diff --git a/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql b/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql index 08b82e48fa..8b25758c4a 100644 --- a/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql +++ b/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql @@ -117,5 +117,6 @@ CREATE TABLE email_inbound_usage_effects ( delivery_id TEXT NOT NULL, finalization_token TEXT NOT NULL, created_at TEXT NOT NULL, - PRIMARY KEY (user_id, delivery_id, finalization_token) + PRIMARY KEY (user_id, delivery_id, finalization_token), + FOREIGN KEY (delivery_id) REFERENCES email_delivery_events(id) ON DELETE CASCADE ); diff --git a/packages/worker/src/email/inbound-delivery-authority.node.test.ts b/packages/worker/src/email/inbound-delivery-authority.node.test.ts index 6b4afd8e6a..0c02036d9f 100644 --- a/packages/worker/src/email/inbound-delivery-authority.node.test.ts +++ b/packages/worker/src/email/inbound-delivery-authority.node.test.ts @@ -1,5 +1,6 @@ import { DatabaseSync } from 'node:sqlite' import { expect, test, vi } from 'vitest' +import { consoleWarn } from '#worker/test-support/console-spies.ts' import { createD1FromSqlite } from '#worker/test-support/create-d1-from-sqlite.ts' import { buildInboundDelivery } from './inbound-delivery.ts' import { @@ -9,7 +10,14 @@ import { import { type MailboxInboundDeliverySnapshot } from './mailbox-inbound-ledger.ts' import { ensureEmailTestSchema } from './test-schema.ts' -test('UserMeter charge precedes Mailbox insert failure and replay does not charge twice', async () => { +function namespace(stub: object) { + return { + idFromName: () => ({}) as DurableObjectId, + get: () => stub, + } as unknown as DurableObjectNamespace +} + +test('dedupe claim precedes UserMeter, and insert failure replay does not double charge', async () => { const sqlite = new DatabaseSync(':memory:') const db = createD1FromSqlite(sqlite) await ensureEmailTestSchema(db) @@ -63,11 +71,6 @@ test('UserMeter charge precedes Mailbox insert failure and replay does not charg return { status: 'inserted' as const, delivery: snapshot } }), } - const namespace = (stub: object) => - ({ - idFromName: () => ({}) as DurableObjectId, - get: () => stub, - }) as unknown as DurableObjectNamespace const authority = createUserInboundDeliveryAuthority({ env: { APP_DB: db, @@ -80,7 +83,7 @@ test('UserMeter charge precedes Mailbox insert failure and replay does not charg await expect( authority.charge({ delivery, plan: 'pro', limit: 100, now }), ).rejects.toThrow('injected Mailbox insert failure') - expect(order).toEqual(['meter-1', 'mailbox-window', 'mailbox-1']) + expect(order).toEqual(['mailbox-window', 'meter-1', 'mailbox-1']) const replay = await authority.charge({ delivery, @@ -90,11 +93,11 @@ test('UserMeter charge precedes Mailbox insert failure and replay does not charg }) expect(replay).toBe(delivery) expect(order).toEqual([ - 'meter-1', 'mailbox-window', + 'meter-1', 'mailbox-1', - 'meter-2', 'mailbox-window', + 'meter-2', 'mailbox-2', ]) expect(consumeCalls).toBe(2) @@ -111,6 +114,256 @@ test('UserMeter charge precedes Mailbox insert failure and replay does not charg sqlite.close() }) +test('dedupe boundary race charges and inserts only the claimed winner id', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const userId = `user-${crypto.randomUUID()}` + const now = new Date('2026-08-02T12:00:00.000Z') + const winner = await buildInboundDelivery({ + userId, + inboxId: `inbox-${crypto.randomUUID()}`, + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'From: sender@example.com\r\n\r\nsame message', + quotaDay: '2026-08-02', + now, + }) + const loser: typeof winner = { + ...winner, + deliveryId: `${winner.deliveryId}-boundary-loser`, + messageId: `${winner.messageId}-boundary-loser`, + rawMimeKey: `${winner.rawMimeKey}-boundary-loser`, + } + const winnerSnapshot: MailboxInboundDeliverySnapshot = { + ...winner, + state: 'pending', + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + } + let stored: MailboxInboundDeliverySnapshot | null = null + const consumedDeliveryIds: Array = [] + const insertedDeliveryIds: Array = [] + const meter = { + consumeInboundDelivery: vi.fn(async (input: { deliveryId: string }) => { + consumedDeliveryIds.push(input.deliveryId) + return { + outcome: 'ready' as const, + count: 1, + revision: 1, + mirrorUpdatedAt: now.toISOString(), + consumed: true, + replayed: false, + day: '2026-08-02', + resource: 'email_receives_per_day' as const, + } + }), + } + const mailbox = { + getInboundDelivery: vi.fn(async (input: { deliveryId: string }) => + stored?.deliveryId === input.deliveryId ? stored : null, + ), + claimInboundDeliveryWindow: vi.fn(async () => winnerSnapshot), + insertChargedPendingInboundDelivery: vi.fn( + async (input: { delivery: { deliveryId: string } }) => { + insertedDeliveryIds.push(input.delivery.deliveryId) + stored = winnerSnapshot + return { status: 'inserted' as const, delivery: winnerSnapshot } + }, + ), + } + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace(meter), + MAILBOX: namespace(mailbox), + }, + userId, + }) + + const first = await authority.charge({ + delivery: winner, + plan: 'pro', + limit: 100, + now, + }) + const boundaryLoser = await authority.charge({ + delivery: loser, + plan: 'pro', + limit: 100, + now, + }) + + expect(first).toBe(winner) + expect(boundaryLoser.deliveryId).toBe(winner.deliveryId) + expect(consumedDeliveryIds).toEqual([winner.deliveryId]) + expect(insertedDeliveryIds).toEqual([winner.deliveryId]) + expect(consumedDeliveryIds).not.toContain(loser.deliveryId) + sqlite.close() +}) + +test('storage claim projection failure releases the authoritative Mailbox lease', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const now = new Date('2026-08-02T12:00:00.000Z') + const delivery = await buildInboundDelivery({ + userId: 'user-storage-owner', + inboxId: 'inbox-storage', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'storage projection', + quotaDay: '2026-08-02', + now, + }) + const claimed: MailboxInboundDeliverySnapshot = { + ...delivery, + state: 'storing', + storageLease: 'storage-lease-1', + storageLeaseAt: now.toISOString(), + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + } + await mirrorUserInboundDeliverySnapshotToD1({ + db, + userId: 'different-owner', + snapshot: claimed, + }) + const releaseInboundDeliveryStorage = vi.fn(async () => ({ + status: 'released' as const, + delivery: { + ...claimed, + state: 'pending' as const, + storageLease: undefined, + storageLeaseAt: undefined, + }, + })) + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace({}), + MAILBOX: namespace({ + claimInboundDeliveryStorage: vi.fn(async () => ({ + status: 'claimed' as const, + delivery: claimed, + })), + releaseInboundDeliveryStorage, + }), + }, + userId: delivery.userId, + }) + + await expect( + authority.claimStorage(delivery, 2, now.toISOString(), now), + ).rejects.toThrow('owner/provider fence') + expect(releaseInboundDeliveryStorage).toHaveBeenCalledWith({ + ownerId: delivery.userId, + deliveryId: delivery.deliveryId, + storageLease: 'storage-lease-1', + now: now.toISOString(), + }) + sqlite.close() +}) + +test('effect and cleanup claim projection failures warn without stranding leases', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + consoleWarn.mockImplementation(() => {}) + const now = new Date('2026-08-02T12:00:00.000Z') + const delivery = await buildInboundDelivery({ + userId: 'user-claim-owner', + inboxId: 'inbox-claim', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'claim projection', + quotaDay: '2026-08-02', + now, + }) + const base: MailboxInboundDeliverySnapshot = { + ...delivery, + state: 'received', + finalizationToken: 'final-1', + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + } + await mirrorUserInboundDeliverySnapshotToD1({ + db, + userId: 'different-owner', + snapshot: base, + }) + const cleanup = { + ...base, + state: 'cleaning' as const, + cleanupLease: 'cleanup-1', + cleanupLeaseAt: now.toISOString(), + } + const usage = { + ...base, + usageEffectLease: 'usage-1', + usageEffectLeaseAt: now.toISOString(), + } + const subscription = { + ...base, + subscriptionEffectState: 'processing' as const, + subscriptionEffectLease: 'subscription-1', + subscriptionEffectLeaseAt: now.toISOString(), + } + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace({}), + MAILBOX: namespace({ + claimInboundDeliveryCleanup: vi.fn(async () => ({ + status: 'claimed' as const, + delivery: cleanup, + })), + claimInboundUsageEffect: vi.fn(async () => ({ + status: 'claimed' as const, + delivery: usage, + })), + claimInboundSubscriptionEffect: vi.fn(async () => ({ + status: 'claimed' as const, + delivery: subscription, + })), + }), + }, + userId: delivery.userId, + }) + + await expect( + authority.claimCleanup(delivery.deliveryId, now), + ).resolves.toMatchObject({ status: 'claimed' }) + await expect( + authority.claimUsageEffect({ deliveryId: delivery.deliveryId, now }), + ).resolves.toMatchObject({ status: 'claimed' }) + await expect( + authority.claimSubscriptionEffect({ + deliveryId: delivery.deliveryId, + now, + }), + ).resolves.toMatchObject({ status: 'claimed' }) + expect(consoleWarn).toHaveBeenCalledWith( + 'inbound-email-cleanup-claim-projection-failed', + delivery.userId, + delivery.deliveryId, + expect.objectContaining({ message: expect.stringContaining('fence') }), + ) + expect(consoleWarn).toHaveBeenCalledWith( + 'inbound-email-usage-effect-claim-projection-failed', + delivery.userId, + delivery.deliveryId, + expect.any(Error), + ) + expect(consoleWarn).toHaveBeenCalledWith( + 'inbound-email-subscription-effect-claim-projection-failed', + delivery.userId, + delivery.deliveryId, + expect.any(Error), + ) + sqlite.close() +}) + test('full D1 snapshot mirror cannot cross an owner or provider fence', async () => { const sqlite = new DatabaseSync(':memory:') const db = createD1FromSqlite(sqlite) diff --git a/packages/worker/src/email/inbound-delivery-authority.ts b/packages/worker/src/email/inbound-delivery-authority.ts index c24e7732ea..eabce07d85 100644 --- a/packages/worker/src/email/inbound-delivery-authority.ts +++ b/packages/worker/src/email/inbound-delivery-authority.ts @@ -8,32 +8,178 @@ import { type UserMeterEnv, } from '#worker/entitlements/user-meter-client.ts' import { - getInboundDelivery as getD1InboundDelivery, - getInboundDeliveryWindow as getD1InboundDeliveryWindow, InboundDeliveryLeaseLostError, type InboundDelivery, } from './inbound-delivery.ts' import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' import { - detailJsonFromMailboxInboundSnapshot, + type MailboxClaimInboundSubscriptionEffectResult, + type MailboxClaimInboundUsageEffectResult, + type MailboxCompleteInboundSubscriptionEffectResult, + type MailboxCompleteInboundUsageEffectResult, + type MailboxFailInboundSubscriptionEffectResult, + type MailboxListDueInboundEffectWorkResult, +} from './mailbox-inbound-effect-ledger.ts' +import { mailboxInboundDedupePointerId, mailboxInboundDedupeProvider, - mailboxInboundProvider, - needsMailboxEffectReconcile, + type MailboxClaimInboundDeliveryCleanupResult, + type MailboxDeferInboundDeliveryReconcileResult, type MailboxInboundDeliveryInsertInput, type MailboxInboundDeliverySnapshot, + type MailboxListDueStaleInboundDeliveriesResult, + type MailboxMarkInboundDeliveryOrphanCleanedResult, + type MailboxReleaseInboundDeliveryCleanupResult, } from './mailbox-inbound-ledger.ts' import { - type MailboxDeliveryEventInput, - type MailboxInboundDeliveryState, -} from './mailbox-types.ts' + bootstrapUserInboundDeliveryFromD1, + bootstrapUserInboundDeliveryWindowFromD1, + mirrorUserInboundDeliverySnapshotToD1, + toUserInboundDelivery, +} from './inbound-delivery-projection.ts' import { systemEmailOwnerId } from './email-owner.ts' +export { mirrorUserInboundDeliverySnapshotToD1 } from './inbound-delivery-projection.ts' + export type UserInboundDeliveryAuthorityEnv = { APP_DB: D1Database } & MailboxEnv & UserMeterEnv +export type CreateUserInboundDeliveryAuthorityInput = { + env: UserInboundDeliveryAuthorityEnv + userId: string +} + +export type UserInboundDeliveryChargeInput = { + delivery: InboundDelivery + plan: PlanName + limit: number + now: Date +} + +export type UserInboundDeliveryReceiveInput = { + delivery: InboundDelivery + usageDurationMs: number + usageMonth: string + usageBytes: number + now?: Date +} + +export type UserInboundDeliveryStorageClaimResult = + | { claimed: true; delivery: InboundDelivery } + | { claimed: false; delivery: InboundDelivery | null } + +export type UserInboundDeliveryOrphanCleanedInput = { + deliveryId: string + cleanupLease: string + outcome: 'deleted' | 'delete-failed' + now?: Date +} + +export type UserInboundUsageEffectClaimInput = { + deliveryId: string + expectedFinalizationToken?: string + now?: Date +} + +export type UserInboundUsageEffectCompleteInput = { + deliveryId: string + usageEffectLease: string + expectedFinalizationToken: string + mode: 'recorded' | 'suppressed' + usageMonth: string + usageBytes: number + usageDurationMs: number + now?: Date +} + +export type UserInboundSubscriptionEffectClaimInput = { + deliveryId: string + expectedFinalizationToken?: string + now?: Date +} + +export type UserInboundSubscriptionEffectCompleteInput = { + deliveryId: string + subscriptionEffectLease: string + expectedFinalizationToken: string + mode: 'complete' | 'suppressed' + suppressionReason?: string + now?: Date +} + +export type UserInboundSubscriptionEffectFailInput = { + deliveryId: string + subscriptionEffectLease: string + expectedFinalizationToken: string + error: string + now?: Date +} + +export type UserInboundDeliveryAuthority = { + get: (deliveryId: string) => Promise + getWindow: (fingerprint: string, now: Date) => Promise + claimWindow: ( + delivery: InboundDelivery, + now: Date, + ) => Promise + charge: (input: UserInboundDeliveryChargeInput) => Promise + claimStorage: ( + delivery: InboundDelivery, + expectedAttachmentCount: number, + usageStartedAt?: string, + now?: Date, + ) => Promise + releaseStorage: (delivery: InboundDelivery, now?: Date) => Promise + reject: ( + delivery: InboundDelivery, + reason: string, + now?: Date, + ) => Promise + receive: (input: UserInboundDeliveryReceiveInput) => Promise + deferReconciliation: ( + deliveryId: string, + now?: Date, + ) => Promise + listDueStale: ( + now?: Date, + limit?: number, + ) => Promise + claimCleanup: ( + deliveryId: string, + now?: Date, + ) => Promise + releaseCleanup: ( + deliveryId: string, + cleanupLease: string, + now?: Date, + ) => Promise + markOrphanCleaned: ( + input: UserInboundDeliveryOrphanCleanedInput, + ) => Promise + pruneExpiredDedupe: (now?: Date, limit?: number) => Promise + claimUsageEffect: ( + input: UserInboundUsageEffectClaimInput, + ) => Promise + completeUsageEffect: ( + input: UserInboundUsageEffectCompleteInput, + ) => Promise + claimSubscriptionEffect: ( + input: UserInboundSubscriptionEffectClaimInput, + ) => Promise + completeSubscriptionEffect: ( + input: UserInboundSubscriptionEffectCompleteInput, + ) => Promise + failSubscriptionEffect: ( + input: UserInboundSubscriptionEffectFailInput, + ) => Promise + listDueEffects: ( + now?: Date, + limit?: number, + ) => Promise +} + function toInsertInput( delivery: InboundDelivery, ): MailboxInboundDeliveryInsertInput { @@ -53,318 +199,9 @@ function toInsertInput( } } -function toInboundDelivery( - userId: string, - snapshot: MailboxInboundDeliverySnapshot, -): InboundDelivery { - const { - createdAt: _createdAt, - updatedAt: _updatedAt, - cleanupRetryAt: _cleanupRetryAt, - reconcileAfter: _reconcileAfter, - ...delivery - } = snapshot - return { ...delivery, userId } -} - -function eventTypeForState(state: MailboxInboundDeliveryState) { - switch (state) { - case 'received': - return 'received' as const - case 'rejected': - return 'rejected' as const - case 'pending': - case 'storing': - case 'cleaning': - case 'orphan-cleaned': - return 'receive_started' as const - default: { - const exhaustive: never = state - throw new Error(`Unhandled inbound delivery state: ${String(exhaustive)}`) - } - } -} - -function toMailboxBootstrapEvent(input: { - delivery: InboundDelivery - eventId: string - provider: string - createdAt: string - updatedAt: string -}): MailboxDeliveryEventInput { - const delivery = input.delivery - const detail = JSON.stringify(delivery) - return { - id: input.eventId, - messageId: delivery.state === 'received' ? delivery.messageId : null, - inboxId: delivery.inboxId, - eventType: eventTypeForState(delivery.state), - provider: input.provider, - providerMessageId: null, - providerEventId: input.eventId, - detailJson: detail, - needsEffectReconcile: - delivery.state === 'received' && needsMailboxEffectReconcile(delivery), - state: delivery.state, - fingerprint: delivery.fingerprint, - storageLease: delivery.storageLease ?? null, - storageLeaseAt: delivery.storageLeaseAt ?? null, - cleanupLease: delivery.cleanupLease ?? null, - cleanupLeaseAt: delivery.cleanupLeaseAt ?? null, - cleanupRetryAt: null, - expectedAttachmentCount: delivery.expectedAttachmentCount ?? null, - finalizationToken: delivery.finalizationToken ?? null, - reconcileAfter: null, - dedupeExpiresAt: delivery.dedupeExpiresAt, - usageEffectRecordedAt: delivery.usageEffectRecordedAt ?? null, - usageEffectSuppressedAt: delivery.usageEffectSuppressedAt ?? null, - usageStartedAt: delivery.usageStartedAt ?? null, - usageMonth: delivery.usageMonth ?? null, - usageBytes: delivery.usageBytes ?? null, - usageDurationMs: delivery.usageDurationMs ?? null, - usageEffectRetryAt: delivery.usageEffectRetryAt ?? null, - usageEffectLease: delivery.usageEffectLease ?? null, - usageEffectLeaseAt: delivery.usageEffectLeaseAt ?? null, - subscriptionEffectState: delivery.subscriptionEffectState ?? null, - subscriptionEffectLease: delivery.subscriptionEffectLease ?? null, - subscriptionEffectLeaseAt: delivery.subscriptionEffectLeaseAt ?? null, - subscriptionEffectRetryAt: delivery.subscriptionEffectRetryAt ?? null, - subscriptionEffectAttemptCount: - delivery.subscriptionEffectAttemptCount ?? null, - subscriptionEffectDeadLetterAt: - delivery.subscriptionEffectDeadLetterAt ?? null, - subscriptionEffectLastError: delivery.subscriptionEffectLastError ?? null, - createdAt: input.createdAt, - updatedAt: input.updatedAt, - } -} - -async function d1EventTimestamps(input: { - db: D1Database - userId: string - eventId: string - provider: string -}) { - return await input.db - .prepare( - `SELECT created_at, COALESCE(updated_at, created_at) AS updated_at - FROM email_delivery_events - WHERE id = ? AND user_id = ? AND provider = ? - LIMIT 1`, - ) - .bind(input.eventId, input.userId, input.provider) - .first<{ created_at: string; updated_at: string }>() -} - -/** - * Full Mailbox → D1 compatibility snapshot. The conflict fence prevents a - * different owner/provider from being overwritten, while updated_at rejects - * delayed mirrors from an older Mailbox CAS. - */ -export async function mirrorUserInboundDeliverySnapshotToD1(input: { - db: D1Database - userId: string - snapshot: MailboxInboundDeliverySnapshot - eventId?: string - provider?: string -}) { - const snapshot = input.snapshot - const eventId = input.eventId ?? snapshot.deliveryId - const provider = input.provider ?? mailboxInboundProvider - const eventType = eventTypeForState(snapshot.state) - const detailJson = detailJsonFromMailboxInboundSnapshot(snapshot, { - userId: input.userId, - }) - const result = await input.db - .prepare( - `INSERT INTO email_delivery_events ( - id, message_id, user_id, inbox_id, event_type, provider, - provider_message_id, provider_event_id, detail_json, - needs_effect_reconcile, state, fingerprint, - storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, - cleanup_retry_at, expected_attachment_count, finalization_token, - reconcile_after, dedupe_expires_at, usage_effect_recorded_at, - usage_effect_suppressed_at, usage_started_at, usage_month, - usage_bytes, usage_duration_ms, usage_effect_retry_at, - usage_effect_lease, usage_effect_lease_at, subscription_effect_state, - subscription_effect_lease, subscription_effect_lease_at, - subscription_effect_retry_at, subscription_effect_attempt_count, - subscription_effect_dead_letter_at, subscription_effect_last_error, - created_at, updated_at - ) VALUES ( - ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? - ) - ON CONFLICT(id) DO UPDATE SET - message_id = excluded.message_id, - inbox_id = excluded.inbox_id, - event_type = excluded.event_type, - provider_event_id = excluded.provider_event_id, - detail_json = excluded.detail_json, - needs_effect_reconcile = excluded.needs_effect_reconcile, - state = excluded.state, - fingerprint = excluded.fingerprint, - storage_lease = excluded.storage_lease, - storage_lease_at = excluded.storage_lease_at, - cleanup_lease = excluded.cleanup_lease, - cleanup_lease_at = excluded.cleanup_lease_at, - cleanup_retry_at = excluded.cleanup_retry_at, - expected_attachment_count = excluded.expected_attachment_count, - finalization_token = excluded.finalization_token, - reconcile_after = excluded.reconcile_after, - dedupe_expires_at = excluded.dedupe_expires_at, - usage_effect_recorded_at = excluded.usage_effect_recorded_at, - usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, - usage_started_at = excluded.usage_started_at, - usage_month = excluded.usage_month, - usage_bytes = excluded.usage_bytes, - usage_duration_ms = excluded.usage_duration_ms, - usage_effect_retry_at = excluded.usage_effect_retry_at, - usage_effect_lease = excluded.usage_effect_lease, - usage_effect_lease_at = excluded.usage_effect_lease_at, - subscription_effect_state = excluded.subscription_effect_state, - subscription_effect_lease = excluded.subscription_effect_lease, - subscription_effect_lease_at = excluded.subscription_effect_lease_at, - subscription_effect_retry_at = excluded.subscription_effect_retry_at, - subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, - subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, - subscription_effect_last_error = excluded.subscription_effect_last_error, - updated_at = excluded.updated_at - WHERE email_delivery_events.user_id = excluded.user_id - AND email_delivery_events.provider = excluded.provider - AND excluded.updated_at >= COALESCE( - email_delivery_events.updated_at, - email_delivery_events.created_at - )`, - ) - .bind( - eventId, - snapshot.state === 'received' ? snapshot.messageId : null, - input.userId, - snapshot.inboxId, - eventType, - provider, - eventId, - detailJson, - snapshot.state === 'received' && needsMailboxEffectReconcile(snapshot) - ? 1 - : 0, - snapshot.state, - snapshot.fingerprint, - snapshot.storageLease ?? null, - snapshot.storageLeaseAt ?? null, - snapshot.cleanupLease ?? null, - snapshot.cleanupLeaseAt ?? null, - snapshot.cleanupRetryAt ?? null, - snapshot.expectedAttachmentCount ?? null, - snapshot.finalizationToken ?? null, - snapshot.reconcileAfter ?? null, - snapshot.dedupeExpiresAt, - snapshot.usageEffectRecordedAt ?? null, - snapshot.usageEffectSuppressedAt ?? null, - snapshot.usageStartedAt ?? null, - snapshot.usageMonth ?? null, - snapshot.usageBytes ?? null, - snapshot.usageDurationMs ?? null, - snapshot.usageEffectRetryAt ?? null, - snapshot.usageEffectLease ?? null, - snapshot.usageEffectLeaseAt ?? null, - snapshot.subscriptionEffectState ?? null, - snapshot.subscriptionEffectLease ?? null, - snapshot.subscriptionEffectLeaseAt ?? null, - snapshot.subscriptionEffectRetryAt ?? null, - snapshot.subscriptionEffectAttemptCount ?? null, - snapshot.subscriptionEffectDeadLetterAt ?? null, - snapshot.subscriptionEffectLastError ?? null, - snapshot.createdAt, - snapshot.updatedAt, - ) - .run() - if (Number(result.meta.changes ?? 0) < 1) { - throw new Error( - 'Mailbox inbound snapshot failed its D1 owner/provider fence.', - ) - } -} - -async function bootstrapDeliveryFromD1(input: { - env: UserInboundDeliveryAuthorityEnv - userId: string - deliveryId: string -}) { - const delivery = await getD1InboundDelivery({ - db: input.env.APP_DB, - userId: input.userId, - deliveryId: input.deliveryId, - }) - if (!delivery) return null - const timestamps = await d1EventTimestamps({ - db: input.env.APP_DB, - userId: input.userId, - eventId: input.deliveryId, - provider: mailboxInboundProvider, - }) - if (!timestamps) return null - const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) - await mailbox.upsertDeliveryEvent({ - ownerId: input.userId, - event: toMailboxBootstrapEvent({ - delivery, - eventId: input.deliveryId, - provider: mailboxInboundProvider, - createdAt: timestamps.created_at, - updatedAt: timestamps.updated_at, - }), - }) - return await mailbox.getInboundDelivery({ - ownerId: input.userId, - deliveryId: input.deliveryId, - }) -} - -async function bootstrapWindowFromD1(input: { - env: UserInboundDeliveryAuthorityEnv - userId: string - fingerprint: string - now: Date -}) { - const delivery = await getD1InboundDeliveryWindow({ - db: input.env.APP_DB, - userId: input.userId, - fingerprint: input.fingerprint, - now: input.now, - }) - if (!delivery) return null - const eventId = mailboxInboundDedupePointerId(input.fingerprint) - const timestamps = await d1EventTimestamps({ - db: input.env.APP_DB, - userId: input.userId, - eventId, - provider: mailboxInboundDedupeProvider, - }) - if (!timestamps) return null - const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) - await mailbox.upsertDeliveryEvent({ - ownerId: input.userId, - event: toMailboxBootstrapEvent({ - delivery, - eventId, - provider: mailboxInboundDedupeProvider, - createdAt: timestamps.created_at, - updatedAt: timestamps.updated_at, - }), - }) - return await mailbox.getInboundDeliveryWindow({ - ownerId: input.userId, - fingerprint: input.fingerprint, - now: input.now.toISOString(), - }) -} - -export function createUserInboundDeliveryAuthority(input: { - env: UserInboundDeliveryAuthorityEnv - userId: string -}) { +export function createUserInboundDeliveryAuthority( + input: CreateUserInboundDeliveryAuthorityInput, +): UserInboundDeliveryAuthority { const { env, userId } = input if (userId === systemEmailOwnerId) { throw new Error( @@ -382,101 +219,121 @@ export function createUserInboundDeliveryAuthority(input: { snapshot, ...options, }) - return toInboundDelivery(userId, snapshot) + return toUserInboundDelivery(userId, snapshot) } - return { - async get(deliveryId: string) { - const current = - (await mailbox.getInboundDelivery({ ownerId: userId, deliveryId })) ?? - (await bootstrapDeliveryFromD1({ env, userId, deliveryId })) - return current ? await mirror(current) : null - }, - async getWindow(fingerprint: string, now: Date) { - const current = - (await mailbox.getInboundDeliveryWindow({ - ownerId: userId, - fingerprint, - now: now.toISOString(), - })) ?? (await bootstrapWindowFromD1({ env, userId, fingerprint, now })) - return current - ? await mirror(current, { - eventId: mailboxInboundDedupePointerId(fingerprint), - provider: mailboxInboundDedupeProvider, - }) - : null - }, - async claimWindow(delivery: InboundDelivery, now: Date) { - const snapshot = await mailbox.claimInboundDeliveryWindow({ + const mirrorClaimBestEffort = async ( + snapshot: MailboxInboundDeliverySnapshot, + logLabel: string, + ) => { + await mirror(snapshot).catch((error: unknown) => { + console.warn(logLabel, userId, snapshot.deliveryId, error) + }) + } + /** + * Mailbox is authoritative. A successful point read synchronously repairs + * the D1 compatibility projection; only a Mailbox miss may invoke the + * one-time pre-deploy bootstrap bridge. + */ + const get = async (deliveryId: string) => { + const current = + (await mailbox.getInboundDelivery({ ownerId: userId, deliveryId })) ?? + (await bootstrapUserInboundDeliveryFromD1({ env, userId, deliveryId })) + return current ? await mirror(current) : null + } + const getWindow = async (fingerprint: string, now: Date) => { + const current = + (await mailbox.getInboundDeliveryWindow({ ownerId: userId, - delivery: toInsertInput(delivery), + fingerprint, now: now.toISOString(), + })) ?? + (await bootstrapUserInboundDeliveryWindowFromD1({ + env, + userId, + fingerprint, + now, + })) + return current + ? await mirror(current, { + eventId: mailboxInboundDedupePointerId(fingerprint), + provider: mailboxInboundDedupeProvider, + }) + : null + } + const claimWindow = async (delivery: InboundDelivery, now: Date) => { + const snapshot = await mailbox.claimInboundDeliveryWindow({ + ownerId: userId, + delivery: toInsertInput(delivery), + now: now.toISOString(), + }) + return await mirror(snapshot, { + eventId: mailboxInboundDedupePointerId(delivery.fingerprint), + provider: mailboxInboundDedupeProvider, + }) + } + const charge = async (chargeInput: UserInboundDeliveryChargeInput) => { + // Resolve the canonical dedupe winner before charging. Concurrent + // boundary candidates therefore consume quota only under the winner id. + const claimedDelivery = await claimWindow( + chargeInput.delivery, + chargeInput.now, + ) + const existing = await get(claimedDelivery.deliveryId) + if (existing) return existing + const updatedAt = chargeInput.now.toISOString() + const meter = userMeterRpc({ env, userId }) + let meterResult = await meter.consumeInboundDelivery({ + deliveryId: claimedDelivery.deliveryId, + resource: 'email_receives_per_day', + day: claimedDelivery.quotaDay, + limit: chargeInput.limit, + updatedAt, + }) + if (meterResult.outcome === 'needs_bootstrap') { + await meter.initialize({ + resource: 'email_receives_per_day', + day: claimedDelivery.quotaDay, + count: 0, + updatedAt, }) - return await mirror(snapshot, { - eventId: mailboxInboundDedupePointerId(delivery.fingerprint), - provider: mailboxInboundDedupeProvider, - }) - }, - async charge(input: { - delivery: InboundDelivery - plan: PlanName - limit: number - now: Date - }) { - const existing = await this.get(input.delivery.deliveryId) - if (existing) return existing - const updatedAt = input.now.toISOString() - const meter = userMeterRpc({ env, userId }) - let meterResult = await meter.consumeInboundDelivery({ - deliveryId: input.delivery.deliveryId, + meterResult = await meter.consumeInboundDelivery({ + deliveryId: claimedDelivery.deliveryId, resource: 'email_receives_per_day', - day: input.delivery.quotaDay, - limit: input.limit, + day: claimedDelivery.quotaDay, + limit: chargeInput.limit, updatedAt, }) if (meterResult.outcome === 'needs_bootstrap') { - await meter.initialize({ - resource: 'email_receives_per_day', - day: input.delivery.quotaDay, - count: 0, - updatedAt, - }) - meterResult = await meter.consumeInboundDelivery({ - deliveryId: input.delivery.deliveryId, - resource: 'email_receives_per_day', - day: input.delivery.quotaDay, - limit: input.limit, - updatedAt, - }) - if (meterResult.outcome === 'needs_bootstrap') { - throw new Error( - 'UserMeter inbound delivery consume still needs bootstrap after initialize.', - ) - } - } - if (!meterResult.consumed && !meterResult.replayed) { - throw new EntitlementLimitError({ - resource: 'email_receives_per_day', - plan: input.plan, - limit: input.limit, - current: meterResult.count, - upgradeHint: buildEntitlementUpgradeHint('email_receives_per_day'), - }) + throw new Error( + 'UserMeter inbound delivery consume still needs bootstrap after initialize.', + ) } - // UserMeter is deliberately the first durable mutation. Only after - // its idempotent claim succeeds may Mailbox claim the dedupe window - // and insert the charged delivery. - const claimedDelivery = await this.claimWindow(input.delivery, input.now) - const result = await mailbox.insertChargedPendingInboundDelivery({ - ownerId: userId, - delivery: toInsertInput(claimedDelivery), - now: updatedAt, + } + if (!meterResult.consumed && !meterResult.replayed) { + throw new EntitlementLimitError({ + resource: 'email_receives_per_day', + plan: chargeInput.plan, + limit: chargeInput.limit, + current: meterResult.count, + upgradeHint: buildEntitlementUpgradeHint('email_receives_per_day'), }) - await mirror(result.delivery) - return result.status === 'inserted' && - claimedDelivery.deliveryId === input.delivery.deliveryId - ? input.delivery - : toInboundDelivery(userId, result.delivery) - }, + } + const result = await mailbox.insertChargedPendingInboundDelivery({ + ownerId: userId, + delivery: toInsertInput(claimedDelivery), + now: updatedAt, + }) + await mirror(result.delivery) + return result.status === 'inserted' && + claimedDelivery.deliveryId === chargeInput.delivery.deliveryId + ? chargeInput.delivery + : toUserInboundDelivery(userId, result.delivery) + } + return { + get, + getWindow, + claimWindow, + charge, async claimStorage( delivery: InboundDelivery, expectedAttachmentCount: number, @@ -491,9 +348,38 @@ export function createUserInboundDeliveryAuthority(input: { now: now.toISOString(), }) if (result.status === 'claimed') { + const storageLease = result.delivery.storageLease + if (!storageLease) { + throw new Error( + 'Mailbox returned a claimed inbound storage delivery without a lease.', + ) + } + let projected: InboundDelivery + try { + projected = await mirror(result.delivery) + } catch (projectionError) { + let compensationError: unknown + try { + await mailbox.releaseInboundDeliveryStorage({ + ownerId: userId, + deliveryId: result.delivery.deliveryId, + storageLease, + now: now.toISOString(), + }) + } catch (error) { + compensationError = error + } + if (compensationError) { + throw new AggregateError( + [projectionError, compensationError], + 'Inbound storage claim projection failed and its Mailbox lease could not be released.', + ) + } + throw projectionError + } return { claimed: true as const, - delivery: await mirror(result.delivery), + delivery: projected, } } return { @@ -535,13 +421,7 @@ export function createUserInboundDeliveryAuthority(input: { 'Inbound rejection lost a state race; delivery should be retried.', ) }, - async receive(input: { - delivery: InboundDelivery - usageDurationMs: number - usageMonth: string - usageBytes: number - now?: Date - }) { + async receive(input: UserInboundDeliveryReceiveInput) { if (!input.delivery.storageLease) { throw new InboundDeliveryLeaseLostError( 'Inbound delivery finalization requires a storage lease.', @@ -588,7 +468,14 @@ export function createUserInboundDeliveryAuthority(input: { deliveryId, now: now.toISOString(), }) - if (result.delivery) await mirror(result.delivery) + if (result.status === 'claimed') { + await mirrorClaimBestEffort( + result.delivery, + 'inbound-email-cleanup-claim-projection-failed', + ) + } else if (result.delivery) { + await mirror(result.delivery) + } return result }, async releaseCleanup( @@ -605,12 +492,7 @@ export function createUserInboundDeliveryAuthority(input: { if (result.status === 'released') await mirror(result.delivery) return result }, - async markOrphanCleaned(input: { - deliveryId: string - cleanupLease: string - outcome: 'deleted' | 'delete-failed' - now?: Date - }) { + async markOrphanCleaned(input: UserInboundDeliveryOrphanCleanedInput) { const result = await mailbox.markInboundDeliveryOrphanCleaned({ ownerId: userId, deliveryId: input.deliveryId, @@ -647,30 +529,24 @@ export function createUserInboundDeliveryAuthority(input: { .run() return result.pruned }, - async claimUsageEffect(input: { - deliveryId: string - expectedFinalizationToken?: string - now?: Date - }) { + async claimUsageEffect(input: UserInboundUsageEffectClaimInput) { const result = await mailbox.claimInboundUsageEffect({ ownerId: userId, deliveryId: input.deliveryId, expectedFinalizationToken: input.expectedFinalizationToken, now: (input.now ?? new Date()).toISOString(), }) - if (result.delivery) await mirror(result.delivery) + if (result.status === 'claimed') { + await mirrorClaimBestEffort( + result.delivery, + 'inbound-email-usage-effect-claim-projection-failed', + ) + } else if (result.delivery) { + await mirror(result.delivery) + } return result }, - async completeUsageEffect(input: { - deliveryId: string - usageEffectLease: string - expectedFinalizationToken: string - mode: 'recorded' | 'suppressed' - usageMonth: string - usageBytes: number - usageDurationMs: number - now?: Date - }) { + async completeUsageEffect(input: UserInboundUsageEffectCompleteInput) { const result = await mailbox.completeInboundUsageEffect({ ownerId: userId, deliveryId: input.deliveryId, @@ -685,28 +561,28 @@ export function createUserInboundDeliveryAuthority(input: { if (result.status !== 'lease-lost') await mirror(result.delivery) return result }, - async claimSubscriptionEffect(input: { - deliveryId: string - expectedFinalizationToken?: string - now?: Date - }) { + async claimSubscriptionEffect( + input: UserInboundSubscriptionEffectClaimInput, + ) { const result = await mailbox.claimInboundSubscriptionEffect({ ownerId: userId, deliveryId: input.deliveryId, expectedFinalizationToken: input.expectedFinalizationToken, now: (input.now ?? new Date()).toISOString(), }) - if (result.delivery) await mirror(result.delivery) + if (result.status === 'claimed') { + await mirrorClaimBestEffort( + result.delivery, + 'inbound-email-subscription-effect-claim-projection-failed', + ) + } else if (result.delivery) { + await mirror(result.delivery) + } return result }, - async completeSubscriptionEffect(input: { - deliveryId: string - subscriptionEffectLease: string - expectedFinalizationToken: string - mode: 'complete' | 'suppressed' - suppressionReason?: string - now?: Date - }) { + async completeSubscriptionEffect( + input: UserInboundSubscriptionEffectCompleteInput, + ) { const result = await mailbox.completeInboundSubscriptionEffect({ ownerId: userId, deliveryId: input.deliveryId, @@ -719,13 +595,9 @@ export function createUserInboundDeliveryAuthority(input: { if (result.status !== 'lease-lost') await mirror(result.delivery) return result }, - async failSubscriptionEffect(input: { - deliveryId: string - subscriptionEffectLease: string - expectedFinalizationToken: string - error: string - now?: Date - }) { + async failSubscriptionEffect( + input: UserInboundSubscriptionEffectFailInput, + ) { const result = await mailbox.failInboundSubscriptionEffect({ ownerId: userId, deliveryId: input.deliveryId, @@ -744,5 +616,5 @@ export function createUserInboundDeliveryAuthority(input: { limit, }) }, - } + } satisfies UserInboundDeliveryAuthority } diff --git a/packages/worker/src/email/inbound-delivery-projection.ts b/packages/worker/src/email/inbound-delivery-projection.ts new file mode 100644 index 0000000000..ebf61eff2a --- /dev/null +++ b/packages/worker/src/email/inbound-delivery-projection.ts @@ -0,0 +1,329 @@ +import { + getInboundDelivery as getD1InboundDelivery, + getInboundDeliveryWindow as getD1InboundDeliveryWindow, + type InboundDelivery, +} from './inbound-delivery.ts' +import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' +import { + detailJsonFromMailboxInboundSnapshot, + mailboxInboundDedupePointerId, + mailboxInboundDedupeProvider, + mailboxInboundProvider, + needsMailboxEffectReconcile, + type MailboxInboundDeliverySnapshot, +} from './mailbox-inbound-ledger.ts' +import { + type MailboxDeliveryEventInput, + type MailboxInboundDeliveryState, +} from './mailbox-types.ts' + +export type UserInboundDeliveryProjectionEnv = { + APP_DB: D1Database +} & MailboxEnv + +export function toUserInboundDelivery( + userId: string, + snapshot: MailboxInboundDeliverySnapshot, +): InboundDelivery { + const { + createdAt: _createdAt, + updatedAt: _updatedAt, + cleanupRetryAt: _cleanupRetryAt, + reconcileAfter: _reconcileAfter, + ...delivery + } = snapshot + return { ...delivery, userId } +} + +function eventTypeForState(state: MailboxInboundDeliveryState) { + switch (state) { + case 'received': + return 'received' as const + case 'rejected': + return 'rejected' as const + case 'pending': + case 'storing': + case 'cleaning': + case 'orphan-cleaned': + return 'receive_started' as const + default: { + const exhaustive: never = state + throw new Error(`Unhandled inbound delivery state: ${String(exhaustive)}`) + } + } +} + +function toMailboxBootstrapEvent(input: { + delivery: InboundDelivery + eventId: string + provider: string + createdAt: string + updatedAt: string +}): MailboxDeliveryEventInput { + const delivery = input.delivery + return { + id: input.eventId, + messageId: delivery.state === 'received' ? delivery.messageId : null, + inboxId: delivery.inboxId, + eventType: eventTypeForState(delivery.state), + provider: input.provider, + providerMessageId: null, + providerEventId: input.eventId, + detailJson: JSON.stringify(delivery), + needsEffectReconcile: + delivery.state === 'received' && needsMailboxEffectReconcile(delivery), + state: delivery.state, + fingerprint: delivery.fingerprint, + storageLease: delivery.storageLease ?? null, + storageLeaseAt: delivery.storageLeaseAt ?? null, + cleanupLease: delivery.cleanupLease ?? null, + cleanupLeaseAt: delivery.cleanupLeaseAt ?? null, + cleanupRetryAt: null, + expectedAttachmentCount: delivery.expectedAttachmentCount ?? null, + finalizationToken: delivery.finalizationToken ?? null, + reconcileAfter: null, + dedupeExpiresAt: delivery.dedupeExpiresAt, + usageEffectRecordedAt: delivery.usageEffectRecordedAt ?? null, + usageEffectSuppressedAt: delivery.usageEffectSuppressedAt ?? null, + usageStartedAt: delivery.usageStartedAt ?? null, + usageMonth: delivery.usageMonth ?? null, + usageBytes: delivery.usageBytes ?? null, + usageDurationMs: delivery.usageDurationMs ?? null, + usageEffectRetryAt: delivery.usageEffectRetryAt ?? null, + usageEffectLease: delivery.usageEffectLease ?? null, + usageEffectLeaseAt: delivery.usageEffectLeaseAt ?? null, + subscriptionEffectState: delivery.subscriptionEffectState ?? null, + subscriptionEffectLease: delivery.subscriptionEffectLease ?? null, + subscriptionEffectLeaseAt: delivery.subscriptionEffectLeaseAt ?? null, + subscriptionEffectRetryAt: delivery.subscriptionEffectRetryAt ?? null, + subscriptionEffectAttemptCount: + delivery.subscriptionEffectAttemptCount ?? null, + subscriptionEffectDeadLetterAt: + delivery.subscriptionEffectDeadLetterAt ?? null, + subscriptionEffectLastError: delivery.subscriptionEffectLastError ?? null, + createdAt: input.createdAt, + updatedAt: input.updatedAt, + } +} + +async function d1EventTimestamps(input: { + db: D1Database + userId: string + eventId: string + provider: string +}) { + return await input.db + .prepare( + `SELECT created_at, COALESCE(updated_at, created_at) AS updated_at + FROM email_delivery_events + WHERE id = ? AND user_id = ? AND provider = ? + LIMIT 1`, + ) + .bind(input.eventId, input.userId, input.provider) + .first<{ created_at: string; updated_at: string }>() +} + +/** + * Full Mailbox → D1 compatibility snapshot. The conflict fence prevents a + * different owner/provider from being overwritten, while updated_at rejects + * delayed mirrors from an older Mailbox CAS. + */ +export async function mirrorUserInboundDeliverySnapshotToD1(input: { + db: D1Database + userId: string + snapshot: MailboxInboundDeliverySnapshot + eventId?: string + provider?: string +}) { + const snapshot = input.snapshot + const eventId = input.eventId ?? snapshot.deliveryId + const provider = input.provider ?? mailboxInboundProvider + const eventType = eventTypeForState(snapshot.state) + const detailJson = detailJsonFromMailboxInboundSnapshot(snapshot, { + userId: input.userId, + }) + const result = await input.db + .prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_message_id, provider_event_id, detail_json, + needs_effect_reconcile, state, fingerprint, + storage_lease, storage_lease_at, cleanup_lease, cleanup_lease_at, + cleanup_retry_at, expected_attachment_count, finalization_token, + reconcile_after, dedupe_expires_at, usage_effect_recorded_at, + usage_effect_suppressed_at, usage_started_at, usage_month, + usage_bytes, usage_duration_ms, usage_effect_retry_at, + usage_effect_lease, usage_effect_lease_at, subscription_effect_state, + subscription_effect_lease, subscription_effect_lease_at, + subscription_effect_retry_at, subscription_effect_attempt_count, + subscription_effect_dead_letter_at, subscription_effect_last_error, + created_at, updated_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ON CONFLICT(id) DO UPDATE SET + message_id = excluded.message_id, + inbox_id = excluded.inbox_id, + event_type = excluded.event_type, + provider_event_id = excluded.provider_event_id, + detail_json = excluded.detail_json, + needs_effect_reconcile = excluded.needs_effect_reconcile, + state = excluded.state, + fingerprint = excluded.fingerprint, + storage_lease = excluded.storage_lease, + storage_lease_at = excluded.storage_lease_at, + cleanup_lease = excluded.cleanup_lease, + cleanup_lease_at = excluded.cleanup_lease_at, + cleanup_retry_at = excluded.cleanup_retry_at, + expected_attachment_count = excluded.expected_attachment_count, + finalization_token = excluded.finalization_token, + reconcile_after = excluded.reconcile_after, + dedupe_expires_at = excluded.dedupe_expires_at, + usage_effect_recorded_at = excluded.usage_effect_recorded_at, + usage_effect_suppressed_at = excluded.usage_effect_suppressed_at, + usage_started_at = excluded.usage_started_at, + usage_month = excluded.usage_month, + usage_bytes = excluded.usage_bytes, + usage_duration_ms = excluded.usage_duration_ms, + usage_effect_retry_at = excluded.usage_effect_retry_at, + usage_effect_lease = excluded.usage_effect_lease, + usage_effect_lease_at = excluded.usage_effect_lease_at, + subscription_effect_state = excluded.subscription_effect_state, + subscription_effect_lease = excluded.subscription_effect_lease, + subscription_effect_lease_at = excluded.subscription_effect_lease_at, + subscription_effect_retry_at = excluded.subscription_effect_retry_at, + subscription_effect_attempt_count = excluded.subscription_effect_attempt_count, + subscription_effect_dead_letter_at = excluded.subscription_effect_dead_letter_at, + subscription_effect_last_error = excluded.subscription_effect_last_error, + updated_at = excluded.updated_at + WHERE email_delivery_events.user_id = excluded.user_id + AND email_delivery_events.provider = excluded.provider + AND excluded.updated_at >= COALESCE( + email_delivery_events.updated_at, + email_delivery_events.created_at + )`, + ) + .bind( + eventId, + snapshot.state === 'received' ? snapshot.messageId : null, + input.userId, + snapshot.inboxId, + eventType, + provider, + eventId, + detailJson, + snapshot.state === 'received' && needsMailboxEffectReconcile(snapshot) + ? 1 + : 0, + snapshot.state, + snapshot.fingerprint, + snapshot.storageLease ?? null, + snapshot.storageLeaseAt ?? null, + snapshot.cleanupLease ?? null, + snapshot.cleanupLeaseAt ?? null, + snapshot.cleanupRetryAt ?? null, + snapshot.expectedAttachmentCount ?? null, + snapshot.finalizationToken ?? null, + snapshot.reconcileAfter ?? null, + snapshot.dedupeExpiresAt, + snapshot.usageEffectRecordedAt ?? null, + snapshot.usageEffectSuppressedAt ?? null, + snapshot.usageStartedAt ?? null, + snapshot.usageMonth ?? null, + snapshot.usageBytes ?? null, + snapshot.usageDurationMs ?? null, + snapshot.usageEffectRetryAt ?? null, + snapshot.usageEffectLease ?? null, + snapshot.usageEffectLeaseAt ?? null, + snapshot.subscriptionEffectState ?? null, + snapshot.subscriptionEffectLease ?? null, + snapshot.subscriptionEffectLeaseAt ?? null, + snapshot.subscriptionEffectRetryAt ?? null, + snapshot.subscriptionEffectAttemptCount ?? null, + snapshot.subscriptionEffectDeadLetterAt ?? null, + snapshot.subscriptionEffectLastError ?? null, + snapshot.createdAt, + snapshot.updatedAt, + ) + .run() + if (Number(result.meta.changes ?? 0) < 1) { + throw new Error( + 'Mailbox inbound snapshot failed its D1 owner/provider fence.', + ) + } +} + +export async function bootstrapUserInboundDeliveryFromD1(input: { + env: UserInboundDeliveryProjectionEnv + userId: string + deliveryId: string +}) { + const delivery = await getD1InboundDelivery({ + db: input.env.APP_DB, + userId: input.userId, + deliveryId: input.deliveryId, + }) + if (!delivery) return null + const timestamps = await d1EventTimestamps({ + db: input.env.APP_DB, + userId: input.userId, + eventId: input.deliveryId, + provider: mailboxInboundProvider, + }) + if (!timestamps) return null + const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) + await mailbox.upsertDeliveryEvent({ + ownerId: input.userId, + event: toMailboxBootstrapEvent({ + delivery, + eventId: input.deliveryId, + provider: mailboxInboundProvider, + createdAt: timestamps.created_at, + updatedAt: timestamps.updated_at, + }), + }) + return await mailbox.getInboundDelivery({ + ownerId: input.userId, + deliveryId: input.deliveryId, + }) +} + +export async function bootstrapUserInboundDeliveryWindowFromD1(input: { + env: UserInboundDeliveryProjectionEnv + userId: string + fingerprint: string + now: Date +}) { + const delivery = await getD1InboundDeliveryWindow({ + db: input.env.APP_DB, + userId: input.userId, + fingerprint: input.fingerprint, + now: input.now, + }) + if (!delivery) return null + const eventId = mailboxInboundDedupePointerId(input.fingerprint) + const timestamps = await d1EventTimestamps({ + db: input.env.APP_DB, + userId: input.userId, + eventId, + provider: mailboxInboundDedupeProvider, + }) + if (!timestamps) return null + const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) + await mailbox.upsertDeliveryEvent({ + ownerId: input.userId, + event: toMailboxBootstrapEvent({ + delivery, + eventId, + provider: mailboxInboundDedupeProvider, + createdAt: timestamps.created_at, + updatedAt: timestamps.updated_at, + }), + }) + return await mailbox.getInboundDeliveryWindow({ + ownerId: input.userId, + fingerprint: input.fingerprint, + now: input.now.toISOString(), + }) +} diff --git a/packages/worker/src/email/inbound-effects.ts b/packages/worker/src/email/inbound-effects.ts index a273b699a1..cc4f15e98f 100644 --- a/packages/worker/src/email/inbound-effects.ts +++ b/packages/worker/src/email/inbound-effects.ts @@ -6,7 +6,7 @@ import { dispatchSystemInboundEmailSubscriptionEvents, } from './package-subscriptions.ts' import { getEmailMessageById } from './repo.ts' -import { systemEmailOwnerId } from './system-email.ts' +import { systemEmailOwnerId } from './email-owner.ts' const subscriptionEffectLeaseMs = 5 * 60 * 1000 const effectRetryMs = 15 * 60 * 1000 @@ -425,7 +425,7 @@ async function processUserInboundDeliveryEffectsWithLeaseHeld(input: { } } -async function processInboundDeliveryEffectsWithLeaseHeld(input: { +export type ProcessInboundDeliveryEffectsInput = { env: InboundEffectsEnv userId: string deliveryId: string @@ -433,7 +433,13 @@ async function processInboundDeliveryEffectsWithLeaseHeld(input: { durationMs?: number now?: Date waitUntil?: (promise: Promise) => void -}) { +} + +// system:email deliberately retains the legacy D1 authority and effect policy; +// USER deliveries use the Mailbox CAS engine below. Keep the split explicit. +async function processSystemInboundDeliveryEffectsWithLeaseHeld( + input: ProcessInboundDeliveryEffectsInput, +) { const now = input.now ?? new Date() const delivery = await getInboundDelivery({ db: input.env.APP_DB, @@ -698,10 +704,10 @@ async function processInboundDeliveryEffectsWithLeaseHeld(input: { } export async function processInboundDeliveryEffects( - input: Parameters[0], + input: ProcessInboundDeliveryEffectsInput, ) { if (input.userId === systemEmailOwnerId) { - return await processInboundDeliveryEffectsWithLeaseHeld(input) + return await processSystemInboundDeliveryEffectsWithLeaseHeld(input) } return await withAccountWriteLease({ db: input.env.APP_DB, diff --git a/packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts b/packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts new file mode 100644 index 0000000000..5281c161a3 --- /dev/null +++ b/packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts @@ -0,0 +1,262 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { DatabaseSync } from 'node:sqlite' +import { expect, test } from 'vitest' + +const migrationsDirectory = new URL('../../migrations/', import.meta.url) +const authorityMirrorMigration = + '0129-email-inbound-mailbox-authority-mirror.sql' + +function applyMigrationsBefore(db: DatabaseSync, exclusiveUpperBound: string) { + for (const fileName of readdirSync(migrationsDirectory) + .filter((file) => file.endsWith('.sql') && file < exclusiveUpperBound) + .sort()) { + db.exec(readFileSync(new URL(fileName, migrationsDirectory), 'utf8')) + } +} + +function applyMigrationLikeD1(db: DatabaseSync, fileName: string) { + const sql = readFileSync(new URL(fileName, migrationsDirectory), 'utf8') + db.exec('BEGIN') + try { + db.exec(sql) + db.exec('COMMIT') + } catch (error) { + db.exec('ROLLBACK') + throw error + } +} + +test('0129 promotes legacy inbound authority and cascades usage idempotency', () => { + using db = new DatabaseSync(':memory:') + applyMigrationsBefore(db, authorityMirrorMigration) + expect( + db + .prepare(`PRAGMA table_info(email_delivery_events)`) + .all() + .some((column) => column.name === 'state'), + ).toBe(false) + + const createdAt = '2026-08-01T00:00:00.000Z' + const insert = db.prepare( + `INSERT INTO email_delivery_events ( + id, user_id, event_type, provider, detail_json, + needs_effect_reconcile, usage_effect_recorded_at, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + insert.run( + 'delivery-backfill', + 'user-backfill', + 'received', + 'cloudflare-email-routing', + JSON.stringify({ + state: 'received', + fingerprint: 'fingerprint-backfill', + storageLease: 'storage-lease', + storageLeaseAt: '2026-08-01T00:01:00.000Z', + cleanupLease: 'cleanup-lease', + cleanupLeaseAt: '2026-08-01T00:02:00.000Z', + cleanupRetryAt: '2026-08-01T00:03:00.000Z', + expectedAttachmentCount: 2, + finalizationToken: 'finalization-token', + reconcileAfter: '2026-08-01T00:04:00.000Z', + dedupeExpiresAt: '2026-08-02T00:00:00.000Z', + usageEffectSuppressedAt: '2026-08-01T00:05:00.000Z', + usageStartedAt: '2026-08-01T00:06:00.000Z', + usageEffectRetryAt: '2026-08-01T00:07:00.000Z', + usageEffectLease: 'usage-lease', + usageEffectLeaseAt: '2026-08-01T00:08:00.000Z', + subscriptionEffectState: 'processing', + subscriptionEffectLease: 'subscription-lease', + subscriptionEffectLeaseAt: '2026-08-01T00:09:00.000Z', + subscriptionEffectRetryAt: '2026-08-01T00:10:00.000Z', + subscriptionEffectAttemptCount: 3, + subscriptionEffectDeadLetterAt: '2026-08-01T00:11:00.000Z', + subscriptionEffectLastError: 'dispatch failed', + }), + 0, + null, + createdAt, + ) + insert.run( + 'usage-null-lease-at', + 'user-usage-due', + 'received', + 'cloudflare-email-routing', + JSON.stringify({ + state: 'received', + fingerprint: 'fingerprint-usage', + usageEffectLease: 'legacy-usage-lease', + subscriptionEffectState: 'complete', + }), + 1, + null, + createdAt, + ) + insert.run( + 'subscription-null-lease-at', + 'user-subscription-due', + 'received', + 'cloudflare-email-routing', + JSON.stringify({ + state: 'received', + fingerprint: 'fingerprint-subscription', + subscriptionEffectState: 'processing', + subscriptionEffectLease: 'legacy-subscription-lease', + }), + 1, + '2026-08-01T00:01:00.000Z', + createdAt, + ) + insert.run( + 'email-inbound-dedupe:fingerprint-dedupe', + 'user-dedupe', + 'receive_started', + 'cloudflare-email-routing-dedupe', + JSON.stringify({ + state: 'pending', + fingerprint: 'fingerprint-dedupe', + dedupeExpiresAt: '2026-08-01T01:00:00.000Z', + }), + 0, + null, + createdAt, + ) + + db.exec('PRAGMA foreign_keys = ON') + applyMigrationLikeD1(db, authorityMirrorMigration) + + expect( + db + .prepare( + `SELECT + state, fingerprint, storage_lease, storage_lease_at, + cleanup_lease, cleanup_lease_at, cleanup_retry_at, + expected_attachment_count, finalization_token, reconcile_after, + dedupe_expires_at, usage_effect_suppressed_at, usage_started_at, + usage_effect_retry_at, usage_effect_lease, usage_effect_lease_at, + subscription_effect_state, subscription_effect_lease, + subscription_effect_lease_at, subscription_effect_retry_at, + subscription_effect_attempt_count, + subscription_effect_dead_letter_at, + subscription_effect_last_error, updated_at + FROM email_delivery_events + WHERE id = 'delivery-backfill'`, + ) + .get(), + ).toEqual({ + state: 'received', + fingerprint: 'fingerprint-backfill', + storage_lease: 'storage-lease', + storage_lease_at: '2026-08-01T00:01:00.000Z', + cleanup_lease: 'cleanup-lease', + cleanup_lease_at: '2026-08-01T00:02:00.000Z', + cleanup_retry_at: '2026-08-01T00:03:00.000Z', + expected_attachment_count: 2, + finalization_token: 'finalization-token', + reconcile_after: '2026-08-01T00:04:00.000Z', + dedupe_expires_at: '2026-08-02T00:00:00.000Z', + usage_effect_suppressed_at: '2026-08-01T00:05:00.000Z', + usage_started_at: '2026-08-01T00:06:00.000Z', + usage_effect_retry_at: '2026-08-01T00:07:00.000Z', + usage_effect_lease: 'usage-lease', + usage_effect_lease_at: '2026-08-01T00:08:00.000Z', + subscription_effect_state: 'processing', + subscription_effect_lease: 'subscription-lease', + subscription_effect_lease_at: '2026-08-01T00:09:00.000Z', + subscription_effect_retry_at: '2026-08-01T00:10:00.000Z', + subscription_effect_attempt_count: 3, + subscription_effect_dead_letter_at: '2026-08-01T00:11:00.000Z', + subscription_effect_last_error: 'dispatch failed', + updated_at: createdAt, + }) + expect( + db + .prepare( + `SELECT state, fingerprint, dedupe_expires_at, updated_at + FROM email_delivery_events + WHERE id = 'email-inbound-dedupe:fingerprint-dedupe'`, + ) + .get(), + ).toEqual({ + state: 'pending', + fingerprint: 'fingerprint-dedupe', + dedupe_expires_at: '2026-08-01T01:00:00.000Z', + updated_at: createdAt, + }) + + expect( + db + .prepare( + `SELECT name FROM sqlite_schema + WHERE type = 'index' + AND name IN ( + 'idx_email_delivery_events_user_state_created', + 'idx_email_delivery_events_user_dedupe_expires' + ) + ORDER BY name`, + ) + .all(), + ).toEqual([ + { name: 'idx_email_delivery_events_user_dedupe_expires' }, + { name: 'idx_email_delivery_events_user_state_created' }, + ]) + expect( + db + .prepare( + `SELECT id + FROM email_delivery_events + WHERE provider = 'cloudflare-email-routing' + AND event_type = 'received' + AND needs_effect_reconcile = 1 + AND fingerprint IS NOT NULL + AND ( + ( + usage_effect_recorded_at IS NULL + AND usage_effect_suppressed_at IS NULL + AND ( + usage_effect_lease IS NULL + OR usage_effect_lease_at IS NULL + OR usage_effect_lease_at < ? + ) + ) + OR ( + usage_effect_recorded_at IS NOT NULL + AND subscription_effect_state = 'processing' + AND ( + subscription_effect_lease_at IS NULL + OR subscription_effect_lease_at < ? + ) + ) + ) + ORDER BY id`, + ) + .all('2026-08-01T00:30:00.000Z', '2026-08-01T00:30:00.000Z'), + ).toEqual([ + { id: 'subscription-null-lease-at' }, + { id: 'usage-null-lease-at' }, + ]) + + expect( + db.prepare(`PRAGMA foreign_key_list(email_inbound_usage_effects)`).all(), + ).toContainEqual( + expect.objectContaining({ + table: 'email_delivery_events', + from: 'delivery_id', + to: 'id', + on_delete: 'CASCADE', + }), + ) + db.prepare( + `INSERT INTO email_inbound_usage_effects ( + user_id, delivery_id, finalization_token, created_at + ) VALUES (?, ?, ?, ?)`, + ).run('user-backfill', 'delivery-backfill', 'finalization-token', createdAt) + db.prepare( + `DELETE FROM email_delivery_events WHERE id = 'delivery-backfill'`, + ).run() + expect( + db + .prepare(`SELECT COUNT(*) AS count FROM email_inbound_usage_effects`) + .get(), + ).toEqual({ count: 0 }) +}) diff --git a/packages/worker/src/email/inbound-mailbox.node.test.ts b/packages/worker/src/email/inbound-mailbox.node.test.ts index 53caf0c396..ba262dba89 100644 --- a/packages/worker/src/email/inbound-mailbox.node.test.ts +++ b/packages/worker/src/email/inbound-mailbox.node.test.ts @@ -199,3 +199,26 @@ test('rejected terminal schedules authority snapshot repair only', async () => { expect(mocks.processInboundDeliveryEffects).not.toHaveBeenCalled() expect(mocks.authorityGet).toHaveBeenCalledWith('delivery-3') }) + +test('rejected terminal contains and logs projection repair failures', async () => { + resetMocks() + consoleError.mockImplementation(() => {}) + mocks.authorityGet.mockRejectedValueOnce(new Error('projection unavailable')) + const { ctx, waitUntilPromises } = createCapturedWaitUntilContext() + + await scheduleInboundRejectedTerminalWork({ + env: { APP_DB: {} } as unknown as Parameters< + typeof scheduleInboundRejectedTerminalWork + >[0]['env'], + userId: 'user-ddd', + deliveryId: 'delivery-4', + ctx, + }) + + expect(waitUntilPromises).toHaveLength(1) + await expect(Promise.all(waitUntilPromises)).resolves.toEqual([undefined]) + expect(consoleError).toHaveBeenCalledWith( + 'Inbound email rejection projection repair failed', + expect.objectContaining({ message: 'projection unavailable' }), + ) +}) diff --git a/packages/worker/src/email/inbound-mailbox.ts b/packages/worker/src/email/inbound-mailbox.ts index a78b834951..4c84f63a59 100644 --- a/packages/worker/src/email/inbound-mailbox.ts +++ b/packages/worker/src/email/inbound-mailbox.ts @@ -108,7 +108,11 @@ export async function scheduleInboundRejectedTerminalWork( const task = createUserInboundDeliveryAuthority({ env: input.env, userId: input.userId, - }).get(input.deliveryId) + }) + .get(input.deliveryId) + .catch((error: unknown) => { + console.error('Inbound email rejection projection repair failed', error) + }) if (input.ctx) { input.ctx.waitUntil(task) diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index e1300281f6..b61780ca44 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -43,6 +43,7 @@ import { } from './inbound-delivery.ts' import { createUserInboundDeliveryAuthority, + type UserInboundDeliveryAuthority, type UserInboundDeliveryAuthorityEnv, } from './inbound-delivery-authority.ts' import { @@ -92,7 +93,7 @@ async function rejectClaimedInboundDelivery(input: { message: ForwardableEmailMessage delivery: InboundDelivery reason: string - authority?: ReturnType + authority?: UserInboundDeliveryAuthority }) { const transitioned = await ( input.authority @@ -132,7 +133,7 @@ async function parseAndStoreInboundEmail(input: { blobs: R2Bucket delivery: InboundDelivery parsed: Awaited> - authority?: ReturnType + authority?: UserInboundDeliveryAuthority }) { const now = new Date().toISOString() try { diff --git a/packages/worker/src/email/inbound.workers.test.ts b/packages/worker/src/email/inbound.workers.test.ts index 7cd5013b4b..351ae7b9bb 100644 --- a/packages/worker/src/email/inbound.workers.test.ts +++ b/packages/worker/src/email/inbound.workers.test.ts @@ -2678,6 +2678,20 @@ test('scheduled sweep cleans quiet-user orphans and recovers partial commits', a ) .run() + // Match the production 0129 backfill: USER owner discovery reads promoted + // compatibility columns, while only system:email retains legacy JSON reads. + await env.APP_DB.prepare( + `UPDATE email_delivery_events + SET state = json_extract(detail_json, '$.state'), + fingerprint = json_extract(detail_json, '$.fingerprint'), + reconcile_after = json_extract(detail_json, '$.reconcileAfter'), + cleanup_retry_at = json_extract(detail_json, '$.cleanupRetryAt'), + dedupe_expires_at = json_extract(detail_json, '$.dedupeExpiresAt') + WHERE user_id IN (?, ?, ?, ?)`, + ) + .bind(orphanUserId, partialUserId, pointerUserId, deletingUserId) + .run() + expect( await sweepStaleInboundDeliveries({ env: createInboundEnv(), diff --git a/packages/worker/src/email/reconcile-inbound-deliveries.ts b/packages/worker/src/email/reconcile-inbound-deliveries.ts index 02558260c3..1498d43cb4 100644 --- a/packages/worker/src/email/reconcile-inbound-deliveries.ts +++ b/packages/worker/src/email/reconcile-inbound-deliveries.ts @@ -8,7 +8,7 @@ import { pruneUserExpiredInboundDedupePointers, reconcileUserStaleInboundDeliveries, } from './inbound-delivery-reconciliation-authority.ts' -import { systemEmailOwnerId } from './system-email.ts' +import { systemEmailOwnerId } from './email-owner.ts' import { withAccountWriteLease } from '#worker/account/deletion-state.ts' const reconciliationUserBatchSize = 25 @@ -48,64 +48,124 @@ export async function sweepStaleInboundDeliveries(input: { // still performed under one explicit userId. Deferring failed attempts and // verification tombstones moves them behind older untouched users. const rows = await input.env.APP_DB.prepare( - `WITH due_users AS ( - SELECT user_id, created_at AS due_at + `WITH authority(system_owner_id) AS (VALUES (?)), + projected_events AS ( + SELECT email_delivery_events.*, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.reconcileAfter') + ELSE reconcile_after + END AS authority_reconcile_after, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.state') + ELSE state + END AS authority_state, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.cleanupRetryAt') + ELSE cleanup_retry_at + END AS authority_cleanup_retry_at, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.dedupeExpiresAt') + ELSE dedupe_expires_at + END AS authority_dedupe_expires_at, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.fingerprint') + ELSE fingerprint + END AS authority_fingerprint, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.usageEffectRecordedAt') + ELSE usage_effect_recorded_at + END AS authority_usage_recorded_at, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.usageEffectSuppressedAt') + ELSE usage_effect_suppressed_at + END AS authority_usage_suppressed_at, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.usageEffectRetryAt') + ELSE usage_effect_retry_at + END AS authority_usage_retry_at, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.usageEffectLease') + ELSE usage_effect_lease + END AS authority_usage_lease, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.usageEffectLeaseAt') + ELSE usage_effect_lease_at + END AS authority_usage_lease_at, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.subscriptionEffectState') + ELSE subscription_effect_state + END AS authority_subscription_state, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.subscriptionEffectRetryAt') + ELSE subscription_effect_retry_at + END AS authority_subscription_retry_at, + CASE WHEN user_id = system_owner_id + THEN json_extract(detail_json, '$.subscriptionEffectLeaseAt') + ELSE subscription_effect_lease_at + END AS authority_subscription_lease_at FROM email_delivery_events + CROSS JOIN authority + ), + due_users AS ( + SELECT user_id, created_at AS due_at + FROM projected_events WHERE provider = 'cloudflare-email-routing' AND event_type = 'receive_started' AND created_at < ? AND ( - json_extract(detail_json, '$.reconcileAfter') IS NULL - OR json_extract(detail_json, '$.reconcileAfter') <= ? + authority_reconcile_after IS NULL + OR authority_reconcile_after <= ? ) AND ( - json_extract(detail_json, '$.state') != 'orphan-cleaned' - OR json_extract(detail_json, '$.cleanupRetryAt') <= ? + authority_state != 'orphan-cleaned' + OR authority_cleanup_retry_at <= ? ) UNION ALL - SELECT user_id, json_extract(detail_json, '$.dedupeExpiresAt') - FROM email_delivery_events + SELECT user_id, authority_dedupe_expires_at + FROM projected_events WHERE provider = 'cloudflare-email-routing-dedupe' - AND json_extract(detail_json, '$.dedupeExpiresAt') <= ? + AND authority_dedupe_expires_at <= ? UNION ALL SELECT user_id, COALESCE( - json_extract(detail_json, '$.usageEffectRetryAt'), - json_extract(detail_json, '$.subscriptionEffectRetryAt'), + authority_usage_retry_at, + authority_subscription_retry_at, created_at ) - FROM email_delivery_events + FROM projected_events WHERE provider = 'cloudflare-email-routing' AND event_type = 'received' AND needs_effect_reconcile = 1 - AND json_extract(detail_json, '$.fingerprint') IS NOT NULL + AND authority_fingerprint IS NOT NULL AND ( ( - json_extract(detail_json, '$.usageEffectRecordedAt') IS NULL - AND json_extract(detail_json, '$.usageEffectSuppressedAt') IS NULL + authority_usage_recorded_at IS NULL + AND authority_usage_suppressed_at IS NULL AND ( - json_extract(detail_json, '$.usageEffectRetryAt') IS NULL - OR json_extract(detail_json, '$.usageEffectRetryAt') <= ? + authority_usage_retry_at IS NULL + OR authority_usage_retry_at <= ? ) AND ( - json_extract(detail_json, '$.usageEffectLease') IS NULL - OR json_extract(detail_json, '$.usageEffectLeaseAt') < ? + authority_usage_lease IS NULL + OR authority_usage_lease_at IS NULL + OR authority_usage_lease_at < ? ) ) OR ( ( - json_extract(detail_json, '$.subscriptionEffectState') IS NULL - OR json_extract( - detail_json, - '$.subscriptionEffectState' - ) NOT IN ('complete', 'dead-letter') + authority_subscription_state IS NULL + OR authority_subscription_state NOT IN ( + 'complete', + 'dead-letter' + ) ) AND ( - json_extract(detail_json, '$.subscriptionEffectRetryAt') IS NULL - OR json_extract(detail_json, '$.subscriptionEffectRetryAt') <= ? + authority_subscription_retry_at IS NULL + OR authority_subscription_retry_at <= ? ) AND ( - json_extract(detail_json, '$.subscriptionEffectState') != 'processing' - OR json_extract(detail_json, '$.subscriptionEffectLeaseAt') < ? + authority_subscription_state != 'processing' + OR authority_subscription_lease_at IS NULL + OR authority_subscription_lease_at < ? ) ) ) @@ -117,6 +177,7 @@ export async function sweepStaleInboundDeliveries(input: { LIMIT ?`, ) .bind( + systemEmailOwnerId, cutoff, now.toISOString(), now.toISOString(), diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index cff2eab5c5..97d224fba6 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -9,6 +9,7 @@ import { markInboundDeliveryReceived, type InboundDelivery, } from './inbound-delivery.ts' +import { type UserInboundDeliveryAuthority } from './inbound-delivery-authority.ts' import { mirrorMailboxMessageGraphFromD1, type MailboxLiveMirrorEnv, @@ -353,15 +354,7 @@ export async function storeIdempotentInboundEmail(input: { parsed: ParsedInboundEmail subjectNormalized: string now: string - authority?: { - get: (deliveryId: string) => Promise - receive: (input: { - delivery: InboundDelivery - usageDurationMs: number - usageMonth: string - usageBytes: number - }) => Promise - } + authority?: Pick }) { const { delivery, parsed } = input if (!delivery.storageLease) { diff --git a/packages/worker/src/email/test-schema.ts b/packages/worker/src/email/test-schema.ts index c3ff0beb15..84a2425a1d 100644 --- a/packages/worker/src/email/test-schema.ts +++ b/packages/worker/src/email/test-schema.ts @@ -199,7 +199,8 @@ WHERE provider = 'cloudflare-email-routing' delivery_id TEXT NOT NULL, finalization_token TEXT NOT NULL, created_at TEXT NOT NULL, - PRIMARY KEY (user_id, delivery_id, finalization_token) + PRIMARY KEY (user_id, delivery_id, finalization_token), + FOREIGN KEY (delivery_id) REFERENCES email_delivery_events(id) ON DELETE CASCADE );`, `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_recorded_usage_month ON email_delivery_events(usage_month, user_id) diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index d58462fc96..d871923d4e 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -530,7 +530,7 @@ }, { "filename": "0129-email-inbound-mailbox-authority-mirror.sql", - "sha256": "41df09101819cf992d33504013ebf016d7bbbf49b310d08a02a71cb6b5b6dfe1" + "sha256": "18ba77426f929819989c450ec5e650438a74ce692f3bd73197a46f843412028e" } ] } From 152b682cab2c64c95a70c55cb911ee9c00c5d9d7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 16:50:48 +0000 Subject: [PATCH 56/62] fix(email): fence inbound cleanup projection Co-authored-by: Kent C. Dodds --- .../inbound-delivery-authority.node.test.ts | 240 ++++++++++++++++-- .../src/email/inbound-delivery-authority.ts | 48 +++- 2 files changed, 257 insertions(+), 31 deletions(-) diff --git a/packages/worker/src/email/inbound-delivery-authority.node.test.ts b/packages/worker/src/email/inbound-delivery-authority.node.test.ts index 0c02036d9f..e64fc28fc0 100644 --- a/packages/worker/src/email/inbound-delivery-authority.node.test.ts +++ b/packages/worker/src/email/inbound-delivery-authority.node.test.ts @@ -202,6 +202,97 @@ test('dedupe boundary race charges and inserts only the claimed winner id', asyn sqlite.close() }) +test("uncharged prior-day dedupe winner charges the current retry's quota day", async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const userId = `user-${crypto.randomUUID()}` + const yesterday = new Date('2026-08-01T23:59:00.000Z') + const retryNow = new Date('2026-08-02T00:01:00.000Z') + const winner = await buildInboundDelivery({ + userId, + inboxId: `inbox-${crypto.randomUUID()}`, + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'From: sender@example.com\r\n\r\nmidnight retry', + quotaDay: '2026-08-01', + now: yesterday, + }) + const retry = { ...winner, quotaDay: '2026-08-02' } + const winnerSnapshot: MailboxInboundDeliverySnapshot = { + ...winner, + state: 'pending', + createdAt: yesterday.toISOString(), + updatedAt: retryNow.toISOString(), + } + const consumeInboundDelivery = vi.fn( + async (input: { deliveryId: string; day: string }) => ({ + outcome: 'ready' as const, + count: 1, + revision: 1, + mirrorUpdatedAt: retryNow.toISOString(), + consumed: true, + replayed: false, + day: input.day, + resource: 'email_receives_per_day' as const, + }), + ) + const insertChargedPendingInboundDelivery = vi.fn( + async (input: { + delivery: { + deliveryId: string + messageId: string + rawMimeKey: string + quotaDay: string + } + }) => ({ + status: 'inserted' as const, + delivery: { + ...winnerSnapshot, + quotaDay: input.delivery.quotaDay, + }, + }), + ) + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace({ consumeInboundDelivery }), + MAILBOX: namespace({ + getInboundDelivery: vi.fn(async () => null), + claimInboundDeliveryWindow: vi.fn(async () => winnerSnapshot), + insertChargedPendingInboundDelivery, + }), + }, + userId, + }) + + const result = await authority.charge({ + delivery: retry, + plan: 'pro', + limit: 100, + now: retryNow, + }) + + expect(result).toBe(retry) + expect(consumeInboundDelivery).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryId: winner.deliveryId, + day: '2026-08-02', + }), + ) + expect(insertChargedPendingInboundDelivery).toHaveBeenCalledWith( + expect.objectContaining({ + delivery: expect.objectContaining({ + deliveryId: winner.deliveryId, + messageId: winner.messageId, + rawMimeKey: winner.rawMimeKey, + quotaDay: '2026-08-02', + }), + }), + ) + sqlite.close() +}) + test('storage claim projection failure releases the authoritative Mailbox lease', async () => { const sqlite = new DatabaseSync(':memory:') const db = createD1FromSqlite(sqlite) @@ -265,11 +356,10 @@ test('storage claim projection failure releases the authoritative Mailbox lease' sqlite.close() }) -test('effect and cleanup claim projection failures warn without stranding leases', async () => { +test('cleanup projection failure releases its lease and prevents deletion work', async () => { const sqlite = new DatabaseSync(':memory:') const db = createD1FromSqlite(sqlite) await ensureEmailTestSchema(db) - consoleWarn.mockImplementation(() => {}) const now = new Date('2026-08-02T12:00:00.000Z') const delivery = await buildInboundDelivery({ userId: 'user-claim-owner', @@ -280,6 +370,133 @@ test('effect and cleanup claim projection failures warn without stranding leases quotaDay: '2026-08-02', now, }) + const cleanup: MailboxInboundDeliverySnapshot = { + ...delivery, + state: 'cleaning', + cleanupLease: 'cleanup-1', + cleanupLeaseAt: now.toISOString(), + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + } + await mirrorUserInboundDeliverySnapshotToD1({ + db, + userId: 'different-owner', + snapshot: cleanup, + }) + const releaseInboundDeliveryCleanup = vi.fn(async () => ({ + status: 'released' as const, + delivery: { + ...cleanup, + state: 'pending' as const, + cleanupLease: undefined, + cleanupLeaseAt: undefined, + }, + })) + const deleteBlob = vi.fn(async () => {}) + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace({}), + MAILBOX: namespace({ + claimInboundDeliveryCleanup: vi.fn(async () => ({ + status: 'claimed' as const, + delivery: cleanup, + })), + releaseInboundDeliveryCleanup, + }), + }, + userId: delivery.userId, + }) + + await expect( + authority + .claimCleanup(delivery.deliveryId, now) + .then(async () => await deleteBlob()), + ).rejects.toThrow('owner/provider fence') + expect(deleteBlob).not.toHaveBeenCalled() + expect(releaseInboundDeliveryCleanup).toHaveBeenCalledWith({ + ownerId: delivery.userId, + deliveryId: delivery.deliveryId, + cleanupLease: 'cleanup-1', + now: now.toISOString(), + }) + sqlite.close() +}) + +test('cleanup projection and compensation failures surface as AggregateError', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const now = new Date('2026-08-02T12:00:00.000Z') + const delivery = await buildInboundDelivery({ + userId: 'user-cleanup-compensation-owner', + inboxId: 'inbox-cleanup-compensation', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'cleanup compensation', + quotaDay: '2026-08-02', + now, + }) + const cleanup: MailboxInboundDeliverySnapshot = { + ...delivery, + state: 'cleaning', + cleanupLease: 'cleanup-compensation-lease', + cleanupLeaseAt: now.toISOString(), + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + } + await mirrorUserInboundDeliverySnapshotToD1({ + db, + userId: 'different-owner', + snapshot: cleanup, + }) + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace({}), + MAILBOX: namespace({ + claimInboundDeliveryCleanup: vi.fn(async () => ({ + status: 'claimed' as const, + delivery: cleanup, + })), + releaseInboundDeliveryCleanup: vi.fn(async () => { + throw new Error('cleanup release unavailable') + }), + }), + }, + userId: delivery.userId, + }) + + const error = await authority + .claimCleanup(delivery.deliveryId, now) + .catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(AggregateError) + expect(error).toMatchObject({ + message: + 'Inbound cleanup claim projection failed and its Mailbox lease could not be released.', + errors: [ + expect.objectContaining({ message: expect.stringContaining('fence') }), + expect.objectContaining({ message: 'cleanup release unavailable' }), + ], + }) + sqlite.close() +}) + +test('effect claim projection failures warn without stranding leases', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + consoleWarn.mockImplementation(() => {}) + const now = new Date('2026-08-02T12:00:00.000Z') + const delivery = await buildInboundDelivery({ + userId: 'user-effect-claim-owner', + inboxId: 'inbox-effect-claim', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'effect claim projection', + quotaDay: '2026-08-02', + now, + }) const base: MailboxInboundDeliverySnapshot = { ...delivery, state: 'received', @@ -292,12 +509,6 @@ test('effect and cleanup claim projection failures warn without stranding leases userId: 'different-owner', snapshot: base, }) - const cleanup = { - ...base, - state: 'cleaning' as const, - cleanupLease: 'cleanup-1', - cleanupLeaseAt: now.toISOString(), - } const usage = { ...base, usageEffectLease: 'usage-1', @@ -314,10 +525,6 @@ test('effect and cleanup claim projection failures warn without stranding leases APP_DB: db, USER_METER: namespace({}), MAILBOX: namespace({ - claimInboundDeliveryCleanup: vi.fn(async () => ({ - status: 'claimed' as const, - delivery: cleanup, - })), claimInboundUsageEffect: vi.fn(async () => ({ status: 'claimed' as const, delivery: usage, @@ -331,9 +538,6 @@ test('effect and cleanup claim projection failures warn without stranding leases userId: delivery.userId, }) - await expect( - authority.claimCleanup(delivery.deliveryId, now), - ).resolves.toMatchObject({ status: 'claimed' }) await expect( authority.claimUsageEffect({ deliveryId: delivery.deliveryId, now }), ).resolves.toMatchObject({ status: 'claimed' }) @@ -343,12 +547,6 @@ test('effect and cleanup claim projection failures warn without stranding leases now, }), ).resolves.toMatchObject({ status: 'claimed' }) - expect(consoleWarn).toHaveBeenCalledWith( - 'inbound-email-cleanup-claim-projection-failed', - delivery.userId, - delivery.deliveryId, - expect.objectContaining({ message: expect.stringContaining('fence') }), - ) expect(consoleWarn).toHaveBeenCalledWith( 'inbound-email-usage-effect-claim-projection-failed', delivery.userId, diff --git a/packages/worker/src/email/inbound-delivery-authority.ts b/packages/worker/src/email/inbound-delivery-authority.ts index eabce07d85..4648b294db 100644 --- a/packages/worker/src/email/inbound-delivery-authority.ts +++ b/packages/worker/src/email/inbound-delivery-authority.ts @@ -280,26 +280,30 @@ export function createUserInboundDeliveryAuthority( ) const existing = await get(claimedDelivery.deliveryId) if (existing) return existing + const chargedDelivery = { + ...claimedDelivery, + quotaDay: chargeInput.delivery.quotaDay, + } const updatedAt = chargeInput.now.toISOString() const meter = userMeterRpc({ env, userId }) let meterResult = await meter.consumeInboundDelivery({ - deliveryId: claimedDelivery.deliveryId, + deliveryId: chargedDelivery.deliveryId, resource: 'email_receives_per_day', - day: claimedDelivery.quotaDay, + day: chargedDelivery.quotaDay, limit: chargeInput.limit, updatedAt, }) if (meterResult.outcome === 'needs_bootstrap') { await meter.initialize({ resource: 'email_receives_per_day', - day: claimedDelivery.quotaDay, + day: chargedDelivery.quotaDay, count: 0, updatedAt, }) meterResult = await meter.consumeInboundDelivery({ - deliveryId: claimedDelivery.deliveryId, + deliveryId: chargedDelivery.deliveryId, resource: 'email_receives_per_day', - day: claimedDelivery.quotaDay, + day: chargedDelivery.quotaDay, limit: chargeInput.limit, updatedAt, }) @@ -320,7 +324,7 @@ export function createUserInboundDeliveryAuthority( } const result = await mailbox.insertChargedPendingInboundDelivery({ ownerId: userId, - delivery: toInsertInput(claimedDelivery), + delivery: toInsertInput(chargedDelivery), now: updatedAt, }) await mirror(result.delivery) @@ -469,10 +473,34 @@ export function createUserInboundDeliveryAuthority( now: now.toISOString(), }) if (result.status === 'claimed') { - await mirrorClaimBestEffort( - result.delivery, - 'inbound-email-cleanup-claim-projection-failed', - ) + const cleanupLease = result.delivery.cleanupLease + if (!cleanupLease) { + throw new Error( + 'Mailbox returned a claimed inbound cleanup delivery without a lease.', + ) + } + try { + await mirror(result.delivery) + } catch (projectionError) { + let compensationError: unknown + try { + await mailbox.releaseInboundDeliveryCleanup({ + ownerId: userId, + deliveryId: result.delivery.deliveryId, + cleanupLease, + now: now.toISOString(), + }) + } catch (error) { + compensationError = error + } + if (compensationError) { + throw new AggregateError( + [projectionError, compensationError], + 'Inbound cleanup claim projection failed and its Mailbox lease could not be released.', + ) + } + throw projectionError + } } else if (result.delivery) { await mirror(result.delivery) } From 942debc51f239eb69ba9949beb2eb85e8b27c4f3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 17:20:07 +0000 Subject: [PATCH 57/62] fix(email): enforce Mailbox inbound authority boundary Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 151 +++++++++--------- ...email-inbound-mailbox-authority-mirror.sql | 3 + packages/worker/src/account/data-targets.ts | 9 +- .../inbound-delivery-authority.node.test.ts | 112 ++++++++++++- .../src/email/inbound-delivery-authority.ts | 42 +++-- .../src/email/inbound-delivery-projection.ts | 101 +++++++++--- ...bound-delivery-reconciliation-authority.ts | 60 ++++++- packages/worker/src/email/inbound-delivery.ts | 39 ++++- packages/worker/src/email/inbound-effects.ts | 16 +- ...ox-authority-mirror-migration.node.test.ts | 30 +++- .../inbound-mailbox-mirror.workers.test.ts | 4 +- .../src/email/inbound-mailbox.node.test.ts | 27 +++- packages/worker/src/email/inbound-mailbox.ts | 13 +- packages/worker/src/email/inbound.ts | 34 ++-- packages/worker/src/email/mailbox-do.ts | 35 ++-- .../src/email/mailbox-do.workers.test.ts | 40 ++++- .../src/email/mailbox-inbound-bootstrap.ts | 85 ++++++++++ .../email/mailbox-inbound-cleanup-ledger.ts | 56 +++++-- .../src/email/mailbox-inbound-ledger.ts | 7 +- .../mailbox-inbound-ledger.workers.test.ts | 102 +++++++++++- .../worker/src/email/mailbox-live-mirror.ts | 22 ++- packages/worker/src/email/mailbox-mirror.ts | 1 + .../worker/src/email/mailbox-snapshot-repo.ts | 5 +- .../worker/src/email/mailbox-snapshots.ts | 9 +- packages/worker/src/email/mailbox-store.ts | 8 +- packages/worker/src/email/mailbox-types.ts | 2 + .../src/email/reconcile-inbound-deliveries.ts | 14 +- packages/worker/src/email/service.ts | 9 +- .../system-inbound-delivery-authority.ts | 72 +++++++++ packages/worker/src/email/test-schema.ts | 9 ++ tools/migration-ledger.json | 2 +- 31 files changed, 898 insertions(+), 221 deletions(-) create mode 100644 packages/worker/src/email/mailbox-inbound-bootstrap.ts create mode 100644 packages/worker/src/email/system-inbound-delivery-authority.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 1fc27aa246..6ea8b43b5d 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -490,17 +490,15 @@ Raw email MIME payloads live in the `EMAIL_BLOBS` R2 bucket instead of D1. `insertEmailMessage` puts the payload to `EMAIL_BLOBS` before the D1 insert and writes only `raw_mime_key` (never `raw_mime`). On R2 put failure the insert throws `EmailRawMimeStorageError` (a `RetryableInboundStorageError`; no D1 row). -The inbound Worker refunds the daily receive charge and rethrows only typed -pre-commit failures so Cloudflare Email Routing retries without burning quota. -The durable commit boundary is message + attachment rows: thread prework, R2 -put, and D1 message/attachment storage are pre-commit; `touchEmailThread` / -`received` delivery-event writes are post-commit and are logged without throwing -(retry would duplicate mail). If attachment insert fails but message cleanup -cannot remove the row — or the residual-row probe itself fails (ambiguous commit -state) — the handler acknowledges the already-created message (logged, -non-retry) rather than risking a duplicate. Outbound messages pass -`rawMime: null` and are unaffected. If D1 insert fails after a successful put, -the blob is best-effort deleted. +The inbound Worker rethrows typed pre-commit failures so Cloudflare Email +Routing retries; UserMeter delivery-id idempotency prevents a second charge. The +message-graph commit boundary is message + attachment rows: thread prework, R2 +put, and D1 message/attachment storage precede Mailbox `received` finalization. +If attachment insert fails but message cleanup cannot remove the row — or the +residual-row probe itself fails (ambiguous commit state) — the handler +acknowledges the already-created message (logged, non-retry) rather than risking +a duplicate. Outbound messages pass `rawMime: null` and are unaffected. If D1 +insert fails after a successful put, the blob is best-effort deleted. **Expand/contract Stage 4b1 (code-only):** the worker does not read or write transitional `email_messages.raw_mime` / `raw_mime_offload_blocked`, does not @@ -862,26 +860,26 @@ Never throws; returns a bounded summary. **Live callers:** transport handlers (`account-email.ts`, `email-message-classify.ts`) delegate here for the D1 mutation + full graph mirror invariant (mirror only after a successful D1 update; failures never change the mutation response). -- **Inbound terminals** (`inbound.ts`) — high-risk user-mail dual-write with - strict ordering: **no Mailbox RPC before** durable D1/R2 message + attachment - storage and `received` finalization win. The winner schedules full graph - repair via `ctx.waitUntil` (`scheduleInboundReceivedTerminalWork`). +- **Inbound terminals** (`inbound.ts`) — USER delivery authority starts in + Mailbox: dedupe claim, UserMeter consume, and charged-pending CAS precede + D1/R2 message-graph storage. Mailbox then finalizes `received` or `rejected` + and synchronously projects the full delivery snapshot to D1. A received winner + schedules D1 message-graph repair via `ctx.waitUntil` + (`scheduleInboundReceivedTerminalWork`) without D1 delivery-event write-back. **Already-received** Email Routing retries (delivery ledger `state === 'received'` with an existing message row) idempotently repair the graph and re-run effect reconciliation without a second charge. **Rejected terminals** (post-claim parse failure or replay of a claimed `rejected` - delivery) mirror the delivery event only via - `scheduleInboundRejectedTerminalWork` (`mirrorMailboxDeliveryEventFromD1`). - After successful `processInboundDeliveryEffects`, the same received - coordinator task re-mirrors the updated delivery event (usage/subscription - fields). Mirror failures, timeouts, and hangs never affect SMTP - reject/refund/retry/charge semantics. **Pre-claim bounded rejection rows** - (`recordBoundedEmailRejectionEvent` for verification, suspension, - sender-policy, size, entitlement, and system-limit gates before delivery - claim/charge) stay **D1-only on the live path** — the every-5-minute - `mailbox_parity` lane backfills them. **`system:email` stays excluded** (no - per-user Mailbox object). Retention sweeper deletes and other bulk - metadata-delete mirrors are **still not wired** on live paths. Scheduled + delivery) read-repair the Mailbox → D1 projection via + `scheduleInboundRejectedTerminalWork`. Effects claim and complete/fail in + Mailbox around external work; terminal snapshots synchronously repair D1. + Terminal coordinator failures are contained after the authoritative CAS. + **Pre-claim bounded rejection rows** (`recordBoundedEmailRejectionEvent` for + verification, suspension, sender-policy, size, entitlement, and system-limit + gates before delivery claim/charge) stay **D1-only on the live path** — the + every-5-minute `mailbox_parity` lane backfills them. **`system:email` stays + excluded** (no per-user Mailbox object). Retention sweeper deletes and other + bulk metadata-delete mirrors are **still not wired** on live paths. Scheduled parity reconcile **is** wired (see below); read cutover is prepared but not flipped. @@ -1059,11 +1057,11 @@ mail content. Account export pages Mailbox state through the `mailbox` section ### Expand/contract phases -This is an expand/contract migration. **Phase 2 live paths are wired (terminal -inbound + parity):** live dual-write covers outbound terminal -message/thread/attachment/event graphs, provider delivery-queue graph repair -(`recorded` / `duplicate` / `stale` with a message, via `waitUntil`), user -classification (full graph repair after D1 update via +This is an expand/contract migration. **Phase 2 live paths are wired (USER +inbound authority + graph dual-write + parity):** live dual-write covers +outbound terminal message/thread/attachment/event graphs, provider +delivery-queue graph repair (`recorded` / `duplicate` / `stale` with a message, +via `waitUntil`), user classification (full graph repair after D1 update via `service.ts#setEmailMessageClassification`), high-risk inbound terminal paths (received graph + rejected delivery-event mirror + post-effects event re-mirror; `waitUntil`; no Mailbox before D1/R2 finalization; pre-claim bounded rejections @@ -1079,12 +1077,14 @@ rejection rows), durable content-watermark replays, compares owner-scoped D1 vs Mailbox counts, persists soak state on `users` (migration `0125`), re-purges the DO when account deletion races the lane, and repairs delete drift via purge/rebuild. Direct delete wiring is pending. D1 remains write authority for -all live paths; owner-facing reads may use Mailbox when the default-off -`mailbox-read-cutover` flag and parity soak pass (phase 3). D1 email rows and -existing R2 inventory deletion stay authoritative during expand. **Phase 2 -contract completion** (every user-mail D1 mutation also writes the DO on live -paths, including retention deletes) remains pending; terminal inbound + parity -cover the high-risk live surface today. +message graphs and non-USER-inbound delivery events; USER inbound lifecycle and +effect transitions are owner-bound Mailbox CAS, synchronously projected to D1. +Owner-facing graph reads may use Mailbox when the default-off +`mailbox-read-cutover` flag and parity soak pass (phase 3). Existing R2 +inventory deletion stays authoritative during expand. **Phase 2 contract +completion** (every user-mail D1 mutation also writes the DO on live paths, +including retention deletes) remains pending; terminal inbound + parity cover +the high-risk live surface today. 1. **Additive scaffold / no live mail behavior** — bind `Mailbox`, freeze `idFromName(userId)`, ship client + `mirrorMessage` / `upsertDeliveryEvent` / @@ -1112,10 +1112,13 @@ cover the high-risk live surface today. The every-5-minute `mailbox_parity` scheduled lane backfills all owner messages and delivery events, durable content-watermark replays, count compares, soak tracking on `users` (migration `0125`), and repairs delete - drift via purge/rebuild. **Still pending for phase-2 contract completion:** - direct delete wiring for explicit/retention deletes (including retention - sweeper metadata-delete mirrors). D1 remains write authority; owner-facing - read cutover is wired behind the default-off flag (phase 3). + drift via purge/rebuild. USER inbound delivery lifecycle/effect state is the + exception: Mailbox is authoritative and D1 is its synchronous compatibility + projection. **Still pending for phase-2 contract completion:** direct delete + wiring for explicit/retention deletes (including retention sweeper + metadata-delete mirrors). D1 remains write authority for the message graph + and non-USER-inbound events; owner-facing read cutover is wired behind the + default-off flag (phase 3). 3. **Owner-facing reads cut over after production soak** — app inbox/detail and MCP list/get/search/attachment/delivery-event reads move to the DO only after production parity soak is verified (`mailbox_parity_matching_since` ≥ 2h @@ -1175,42 +1178,34 @@ below. verification. This phase does not flip Mailbox write authority; contextless provider-id reverse lookups must not enumerate per-user Mailbox objects. -### Inbound durability boundary (D1-authoritative dual-write) - -Today's D1-authoritative inbound commit boundary is documented under -[R2 (`EMAIL_BLOBS`)](#r2-community_assets-email_blobs): thread prework, R2 put, -D1 message/attachment rows are pre-commit; `touchEmailThread` / `received` -delivery-event writes are post-commit best-effort; ambiguous attachment-insert -failures acknowledge rather than risk duplicates. - -**Mailbox dual-write ordering (phase-2 live, D1 still authoritative):** - -- **Pre-commit (no Mailbox):** thread prework, R2 raw-MIME put, D1 - message/attachment storage, and inbound delivery finalization to `received`. -- **Post-commit (best effort, `waitUntil`):** full message graph mirror - (`mirrorMailboxMessageGraphFromD1`) only after the durable commit + - finalization win; already-received retries repair the graph idempotently - without a second charge. Rejected **post-claim** terminals mirror the delivery - event only (`mirrorMailboxDeliveryEventFromD1`). After successful effect - dispatch, re-mirror the updated delivery event. Failures/timeouts never affect - reject, refund, retry, or charge semantics. -- **Pre-claim bounded rejections** (`recordBoundedEmailRejectionEvent` before - delivery claim/charge) write D1 audit rows only on the live path; the - `mailbox_parity` lane backfills them. **`system:email` is excluded** from all - Mailbox mirrors. -- **Ambiguity:** if attachment commit fails but message cleanup (or a residual - probe) cannot prove the pre-commit state, acknowledge the already-created - message (logged, non-retry) rather than risking a duplicate on Email Routing - retry. Empty-thread cleanup stays deferred. - -When Mailbox becomes read-authoritative (phase 3+), the same shape applies with -the DO as the metadata store: - -- **Pre-commit:** thread prework + R2 raw-MIME put + atomic Mailbox - message/attachment commit. -- **Post-commit (best effort):** thread touch and delivery-event writes — log - failures without throwing (retry would duplicate mail). -- **Ambiguity:** same acknowledge-over-retry rule as today. +### Inbound durability boundary (USER Mailbox authority) + +For USER mail, the owner-bound Mailbox ledger is the lifecycle/effect authority: + +1. Mailbox CAS selects the dedupe winner. UserMeter consumes quota for that + winner, then Mailbox inserts the charged pending snapshot. The charged + pending snapshot is synchronously projected to D1. +2. Thread prework, R2 raw-MIME put, and D1 message/attachment storage build the + message graph. D1 remains authoritative for that graph. +3. Mailbox CAS finalizes the delivery as `received` or `rejected`; the complete + Mailbox snapshot is synchronously projected to D1. Fence-critical projection + failures fail closed. +4. Received terminal work repairs the D1-authoritative message graph into + Mailbox without delivery events, runs externally executed effects under + Mailbox leases, then read-repairs the Mailbox → D1 projection. + +Retries inspect Mailbox state and never restore USER delivery authority from D1. +The sole reverse bridge is a missing-row-only bootstrap of a validated, +owner/provider-matched pre-deploy D1 snapshot. Malformed or cross-owner legacy +rows are skipped. Pre-claim bounded rejection audit rows remain D1-only. + +`system:email` is the explicit exception: its inbound lifecycle, effects, and +reconciliation remain D1-authoritative and never bootstrap a Mailbox. + +If attachment commit fails but message cleanup (or a residual probe) cannot +prove the pre-commit state, the handler acknowledges the already-created message +rather than risking a duplicate on Email Routing retry. Empty-thread cleanup +stays deferred. ### Package state model diff --git a/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql b/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql index 8b25758c4a..4d54753ecf 100644 --- a/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql +++ b/packages/worker/migrations/0129-email-inbound-mailbox-authority-mirror.sql @@ -120,3 +120,6 @@ CREATE TABLE email_inbound_usage_effects ( PRIMARY KEY (user_id, delivery_id, finalization_token), FOREIGN KEY (delivery_id) REFERENCES email_delivery_events(id) ON DELETE CASCADE ); + +CREATE INDEX IF NOT EXISTS idx_email_inbound_usage_effects_delivery +ON email_inbound_usage_effects(delivery_id); diff --git a/packages/worker/src/account/data-targets.ts b/packages/worker/src/account/data-targets.ts index dd6d580481..4663cff411 100644 --- a/packages/worker/src/account/data-targets.ts +++ b/packages/worker/src/account/data-targets.ts @@ -156,7 +156,14 @@ export const accountUserDataTargets: ReadonlyArray = [ { kind: 'user_id', table: 'workflow_runs' }, { kind: 'user_id', table: 'package_service_states' }, { kind: 'user_id', table: 'user_storage_buckets' }, - { kind: 'user_id', table: 'email_inbound_usage_effects' }, + { + kind: 'user_id', + table: 'email_inbound_usage_effects', + includeInExport: false, + surface: 'email_inbound_usage_effects', + reason: + 'Internal cross-store effect idempotency ledger with no user-exportable content.', + }, { kind: 'user_id', table: 'usage_rollups' }, { kind: 'user_id', table: 'feature_flag_exposure_rollups' }, { kind: 'user_id', table: 'user_activation_milestones' }, diff --git a/packages/worker/src/email/inbound-delivery-authority.node.test.ts b/packages/worker/src/email/inbound-delivery-authority.node.test.ts index e64fc28fc0..7902c605e3 100644 --- a/packages/worker/src/email/inbound-delivery-authority.node.test.ts +++ b/packages/worker/src/email/inbound-delivery-authority.node.test.ts @@ -8,6 +8,7 @@ import { mirrorUserInboundDeliverySnapshotToD1, } from './inbound-delivery-authority.ts' import { type MailboxInboundDeliverySnapshot } from './mailbox-inbound-ledger.ts' +import { claimSystemInboundDeliveryWindow } from './system-inbound-delivery-authority.ts' import { ensureEmailTestSchema } from './test-schema.ts' function namespace(stub: object) { @@ -91,7 +92,7 @@ test('dedupe claim precedes UserMeter, and insert failure replay does not double limit: 100, now, }) - expect(replay).toBe(delivery) + expect(replay).toEqual({ delivery, charged: true }) expect(order).toEqual([ 'mailbox-window', 'meter-1', @@ -194,8 +195,9 @@ test('dedupe boundary race charges and inserts only the claimed winner id', asyn now, }) - expect(first).toBe(winner) - expect(boundaryLoser.deliveryId).toBe(winner.deliveryId) + expect(first).toEqual({ delivery: winner, charged: true }) + expect(boundaryLoser.delivery.deliveryId).toBe(winner.deliveryId) + expect(boundaryLoser.charged).toBe(false) expect(consumedDeliveryIds).toEqual([winner.deliveryId]) expect(insertedDeliveryIds).toEqual([winner.deliveryId]) expect(consumedDeliveryIds).not.toContain(loser.deliveryId) @@ -273,7 +275,8 @@ test("uncharged prior-day dedupe winner charges the current retry's quota day", now: retryNow, }) - expect(result).toBe(retry) + expect(result.delivery).toEqual(retry) + expect(result.charged).toBe(true) expect(consumeInboundDelivery).toHaveBeenCalledWith( expect.objectContaining({ deliveryId: winner.deliveryId, @@ -410,7 +413,13 @@ test('cleanup projection failure releases its lease and prevents deletion work', await expect( authority - .claimCleanup(delivery.deliveryId, now) + .claimCleanup({ + deliveryId: delivery.deliveryId, + expectedState: delivery.state, + expectedUpdatedAt: cleanup.updatedAt, + staleBefore: new Date(now.getTime() - 1), + now, + }) .then(async () => await deleteBlob()), ).rejects.toThrow('owner/provider fence') expect(deleteBlob).not.toHaveBeenCalled() @@ -468,7 +477,13 @@ test('cleanup projection and compensation failures surface as AggregateError', a }) const error = await authority - .claimCleanup(delivery.deliveryId, now) + .claimCleanup({ + deliveryId: delivery.deliveryId, + expectedState: delivery.state, + expectedUpdatedAt: cleanup.updatedAt, + staleBefore: new Date(now.getTime() - 1), + now, + }) .catch((caught: unknown) => caught) expect(error).toBeInstanceOf(AggregateError) expect(error).toMatchObject({ @@ -612,7 +627,7 @@ test('full D1 snapshot mirror cannot cross an owner or provider fence', async () sqlite.close() }) -test('stale D1 mirrors fail closed without replacing a newer Mailbox snapshot', async () => { +test('stale D1 mirrors are safe no-ops without replacing a newer snapshot', async () => { const sqlite = new DatabaseSync(':memory:') const db = createD1FromSqlite(sqlite) await ensureEmailTestSchema(db) @@ -652,7 +667,7 @@ test('stale D1 mirrors fail closed without replacing a newer Mailbox snapshot', updatedAt: older.toISOString(), }, }), - ).rejects.toThrow('owner/provider fence') + ).resolves.toEqual({ status: 'stale' }) expect( await db .prepare( @@ -679,3 +694,84 @@ test('USER authority refuses the system email owner before bootstrap', () => { }), ).toThrow('must remain in D1') }) + +test('bootstrap skips malformed and cross-owner legacy D1 rows', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + consoleWarn.mockImplementation(() => {}) + const ownerId = `owner-${crypto.randomUUID()}` + const now = new Date('2026-08-02T12:00:00.000Z') + const crossOwner = await buildInboundDelivery({ + userId: `other-${crypto.randomUUID()}`, + inboxId: 'inbox-cross-owner', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'cross owner', + quotaDay: '2026-08-02', + now, + }) + const malformedId = `malformed-${crypto.randomUUID()}` + for (const [id, detailJson] of [ + [crossOwner.deliveryId, JSON.stringify(crossOwner)], + [malformedId, '{'], + ] as const) { + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, user_id, event_type, provider, provider_event_id, + detail_json, created_at + ) VALUES (?, ?, 'receive_started', 'cloudflare-email-routing', ?, ?, ?)`, + ) + .bind(id, ownerId, id, detailJson, now.toISOString()) + .run() + } + const upsertDeliveryEvent = vi.fn() + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace({}), + MAILBOX: namespace({ + getInboundDelivery: vi.fn(async () => null), + upsertDeliveryEvent, + }), + }, + userId: ownerId, + }) + + await expect(authority.get(crossOwner.deliveryId)).resolves.toBeNull() + await expect(authority.get(malformedId)).resolves.toBeNull() + expect(upsertDeliveryEvent).not.toHaveBeenCalled() + expect(consoleWarn).toHaveBeenCalledWith( + 'inbound-email-delivery-bootstrap-row-invalid', + ownerId, + crossOwner.deliveryId, + ) + expect(consoleWarn).toHaveBeenCalledWith( + 'inbound-email-delivery-bootstrap-row-invalid', + ownerId, + malformedId, + ) + sqlite.close() +}) + +test('system D1 mutation surface rejects USER deliveries before writing', async () => { + const now = new Date('2026-08-02T12:00:00.000Z') + const delivery = await buildInboundDelivery({ + userId: 'user-cannot-use-system-d1', + inboxId: 'inbox-system-fence', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + rawMime: 'system fence', + quotaDay: '2026-08-02', + now, + }) + + await expect( + claimSystemInboundDeliveryWindow({ + db: {} as D1Database, + delivery, + now, + }), + ).rejects.toThrow('restricted to system:email') +}) diff --git a/packages/worker/src/email/inbound-delivery-authority.ts b/packages/worker/src/email/inbound-delivery-authority.ts index 4648b294db..31f64c388e 100644 --- a/packages/worker/src/email/inbound-delivery-authority.ts +++ b/packages/worker/src/email/inbound-delivery-authority.ts @@ -58,6 +58,11 @@ export type UserInboundDeliveryChargeInput = { now: Date } +export type UserInboundDeliveryChargeResult = { + delivery: InboundDelivery + charged: boolean +} + export type UserInboundDeliveryReceiveInput = { delivery: InboundDelivery usageDurationMs: number @@ -77,6 +82,14 @@ export type UserInboundDeliveryOrphanCleanedInput = { now?: Date } +export type UserInboundDeliveryCleanupClaimInput = { + deliveryId: string + expectedState: InboundDelivery['state'] + expectedUpdatedAt: string + staleBefore: Date + now?: Date +} + export type UserInboundUsageEffectClaimInput = { deliveryId: string expectedFinalizationToken?: string @@ -124,7 +137,9 @@ export type UserInboundDeliveryAuthority = { delivery: InboundDelivery, now: Date, ) => Promise - charge: (input: UserInboundDeliveryChargeInput) => Promise + charge: ( + input: UserInboundDeliveryChargeInput, + ) => Promise claimStorage: ( delivery: InboundDelivery, expectedAttachmentCount: number, @@ -147,8 +162,7 @@ export type UserInboundDeliveryAuthority = { limit?: number, ) => Promise claimCleanup: ( - deliveryId: string, - now?: Date, + input: UserInboundDeliveryCleanupClaimInput, ) => Promise releaseCleanup: ( deliveryId: string, @@ -279,7 +293,7 @@ export function createUserInboundDeliveryAuthority( chargeInput.now, ) const existing = await get(claimedDelivery.deliveryId) - if (existing) return existing + if (existing) return { delivery: existing, charged: false } const chargedDelivery = { ...claimedDelivery, quotaDay: chargeInput.delivery.quotaDay, @@ -327,11 +341,13 @@ export function createUserInboundDeliveryAuthority( delivery: toInsertInput(chargedDelivery), now: updatedAt, }) - await mirror(result.delivery) - return result.status === 'inserted' && - claimedDelivery.deliveryId === chargeInput.delivery.deliveryId - ? chargeInput.delivery - : toUserInboundDelivery(userId, result.delivery) + const delivery = await mirror(result.delivery) + return { + delivery, + charged: + result.status === 'inserted' && + claimedDelivery.deliveryId === chargeInput.delivery.deliveryId, + } } return { get, @@ -466,10 +482,14 @@ export function createUserInboundDeliveryAuthority( limit, }) }, - async claimCleanup(deliveryId: string, now = new Date()) { + async claimCleanup(input: UserInboundDeliveryCleanupClaimInput) { + const now = input.now ?? new Date() const result = await mailbox.claimInboundDeliveryCleanup({ ownerId: userId, - deliveryId, + deliveryId: input.deliveryId, + expectedState: input.expectedState, + expectedUpdatedAt: input.expectedUpdatedAt, + staleBefore: input.staleBefore.toISOString(), now: now.toISOString(), }) if (result.status === 'claimed') { diff --git a/packages/worker/src/email/inbound-delivery-projection.ts b/packages/worker/src/email/inbound-delivery-projection.ts index ebf61eff2a..629147fc01 100644 --- a/packages/worker/src/email/inbound-delivery-projection.ts +++ b/packages/worker/src/email/inbound-delivery-projection.ts @@ -1,6 +1,5 @@ import { - getInboundDelivery as getD1InboundDelivery, - getInboundDeliveryWindow as getD1InboundDeliveryWindow, + parseStrictInboundDeliveryDetailJson, type InboundDelivery, } from './inbound-delivery.ts' import { mailboxRpc, type MailboxEnv } from './mailbox-client.ts' @@ -16,11 +15,16 @@ import { type MailboxDeliveryEventInput, type MailboxInboundDeliveryState, } from './mailbox-types.ts' +import { systemEmailOwnerId } from './email-owner.ts' export type UserInboundDeliveryProjectionEnv = { APP_DB: D1Database } & MailboxEnv +export type UserInboundDeliveryProjectionResult = + | { status: 'mirrored' } + | { status: 'stale' } + export function toUserInboundDelivery( userId: string, snapshot: MailboxInboundDeliverySnapshot, @@ -114,13 +118,37 @@ async function d1EventTimestamps(input: { }) { return await input.db .prepare( - `SELECT created_at, COALESCE(updated_at, created_at) AS updated_at + `SELECT detail_json, created_at, + COALESCE(updated_at, created_at) AS updated_at FROM email_delivery_events WHERE id = ? AND user_id = ? AND provider = ? LIMIT 1`, ) .bind(input.eventId, input.userId, input.provider) - .first<{ created_at: string; updated_at: string }>() + .first<{ + detail_json: string | null + created_at: string + updated_at: string + }>() +} + +function validatedBootstrapDelivery(input: { + detailJson: unknown + userId: string + deliveryId?: string + fingerprint?: string +}) { + const delivery = parseStrictInboundDeliveryDetailJson(input.detailJson) + if (!delivery) return null + if ( + delivery.userId !== input.userId || + delivery.provider !== mailboxInboundProvider || + (input.deliveryId != null && delivery.deliveryId !== input.deliveryId) || + (input.fingerprint != null && delivery.fingerprint !== input.fingerprint) + ) { + return null + } + return delivery } /** @@ -134,7 +162,7 @@ export async function mirrorUserInboundDeliverySnapshotToD1(input: { snapshot: MailboxInboundDeliverySnapshot eventId?: string provider?: string -}) { +}): Promise { const snapshot = input.snapshot const eventId = input.eventId ?? snapshot.deliveryId const provider = input.provider ?? mailboxInboundProvider @@ -247,11 +275,29 @@ export async function mirrorUserInboundDeliverySnapshotToD1(input: { snapshot.updatedAt, ) .run() - if (Number(result.meta.changes ?? 0) < 1) { + if (Number(result.meta.changes ?? 0) > 0) return { status: 'mirrored' } + const existing = await input.db + .prepare( + `SELECT user_id, provider, COALESCE(updated_at, created_at) AS updated_at + FROM email_delivery_events + WHERE id = ? + LIMIT 1`, + ) + .bind(eventId) + .first<{ user_id: string; provider: string; updated_at: string }>() + if ( + !existing || + existing.user_id !== input.userId || + existing.provider !== provider + ) { throw new Error( 'Mailbox inbound snapshot failed its D1 owner/provider fence.', ) } + if (existing.updated_at >= snapshot.updatedAt) return { status: 'stale' } + throw new Error( + 'Mailbox inbound snapshot was not applied and D1 has no newer projection.', + ) } export async function bootstrapUserInboundDeliveryFromD1(input: { @@ -259,12 +305,7 @@ export async function bootstrapUserInboundDeliveryFromD1(input: { userId: string deliveryId: string }) { - const delivery = await getD1InboundDelivery({ - db: input.env.APP_DB, - userId: input.userId, - deliveryId: input.deliveryId, - }) - if (!delivery) return null + if (input.userId === systemEmailOwnerId) return null const timestamps = await d1EventTimestamps({ db: input.env.APP_DB, userId: input.userId, @@ -272,9 +313,23 @@ export async function bootstrapUserInboundDeliveryFromD1(input: { provider: mailboxInboundProvider, }) if (!timestamps) return null + const delivery = validatedBootstrapDelivery({ + detailJson: timestamps.detail_json, + userId: input.userId, + deliveryId: input.deliveryId, + }) + if (!delivery) { + console.warn( + 'inbound-email-delivery-bootstrap-row-invalid', + input.userId, + input.deliveryId, + ) + return null + } const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) await mailbox.upsertDeliveryEvent({ ownerId: input.userId, + intent: 'user-inbound-bootstrap', event: toMailboxBootstrapEvent({ delivery, eventId: input.deliveryId, @@ -295,13 +350,7 @@ export async function bootstrapUserInboundDeliveryWindowFromD1(input: { fingerprint: string now: Date }) { - const delivery = await getD1InboundDeliveryWindow({ - db: input.env.APP_DB, - userId: input.userId, - fingerprint: input.fingerprint, - now: input.now, - }) - if (!delivery) return null + if (input.userId === systemEmailOwnerId) return null const eventId = mailboxInboundDedupePointerId(input.fingerprint) const timestamps = await d1EventTimestamps({ db: input.env.APP_DB, @@ -310,9 +359,23 @@ export async function bootstrapUserInboundDeliveryWindowFromD1(input: { provider: mailboxInboundDedupeProvider, }) if (!timestamps) return null + const delivery = validatedBootstrapDelivery({ + detailJson: timestamps.detail_json, + userId: input.userId, + fingerprint: input.fingerprint, + }) + if (!delivery || delivery.dedupeExpiresAt <= input.now.toISOString()) { + console.warn( + 'inbound-email-dedupe-bootstrap-row-invalid', + input.userId, + eventId, + ) + return null + } const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) await mailbox.upsertDeliveryEvent({ ownerId: input.userId, + intent: 'user-inbound-bootstrap', event: toMailboxBootstrapEvent({ delivery, eventId, diff --git a/packages/worker/src/email/inbound-delivery-reconciliation-authority.ts b/packages/worker/src/email/inbound-delivery-reconciliation-authority.ts index 4c4eafabbe..4c8c812f45 100644 --- a/packages/worker/src/email/inbound-delivery-reconciliation-authority.ts +++ b/packages/worker/src/email/inbound-delivery-reconciliation-authority.ts @@ -6,8 +6,13 @@ import { } from './inbound-delivery-authority.ts' import { InboundDeliveryLeaseLostError, - type InboundDelivery, + parseStrictInboundDeliveryDetailJson, } from './inbound-delivery.ts' +import { + mailboxInboundDedupePointerId, + mailboxInboundProvider, + mailboxStaleInboundDeliveryAgeMs, +} from './mailbox-inbound-ledger.ts' import { insertEmailAttachments, getEmailMessageById, @@ -46,7 +51,16 @@ async function bootstrapPreDeployDueRows(input: { ) .all<{ id: string }>() const authority = createUserInboundDeliveryAuthority(input) - for (const row of rows.results ?? []) await authority.get(row.id) + for (const row of rows.results ?? []) { + await authority.get(row.id).catch((error: unknown) => { + console.warn( + 'inbound-email-stale-bootstrap-failed', + input.userId, + row.id, + error, + ) + }) + } } async function recoverCommittedDelivery(input: { @@ -151,6 +165,7 @@ export async function reconcileUserStaleInboundDeliveries(input: { }) const authority = createUserInboundDeliveryAuthority(input) const due = await authority.listDueStale(now, staleBatchSize) + const staleBefore = new Date(now.getTime() - mailboxStaleInboundDeliveryAgeMs) let recovered = 0 let cleaned = 0 let budgetExhausted = false @@ -184,12 +199,20 @@ export async function reconcileUserStaleInboundDeliveries(input: { snapshot.deliveryId, error, ) - await authority.deferReconciliation(snapshot.deliveryId, now) + await authority + .deferReconciliation(snapshot.deliveryId, now) + .catch(() => undefined) } continue } - const claim = await authority.claimCleanup(snapshot.deliveryId, now) + const claim = await authority.claimCleanup({ + deliveryId: snapshot.deliveryId, + expectedState: snapshot.state, + expectedUpdatedAt: snapshot.updatedAt, + staleBefore, + now, + }) if (claim.status !== 'claimed') continue const racedMessage = await getEmailMessageById({ db: input.env.APP_DB, @@ -241,7 +264,7 @@ export async function pruneUserExpiredInboundDedupePointers(input: { const now = input.now ?? new Date() const authority = createUserInboundDeliveryAuthority(input) const bridgeRows = await input.env.APP_DB.prepare( - `SELECT detail_json + `SELECT id, detail_json FROM email_delivery_events WHERE user_id = ? AND provider = ? @@ -258,10 +281,31 @@ export async function pruneUserExpiredInboundDedupePointers(input: { now.toISOString(), input.limit ?? 20, ) - .all<{ detail_json: string }>() + .all<{ id: string; detail_json: string | null }>() for (const row of bridgeRows.results ?? []) { - const delivery = JSON.parse(row.detail_json) as InboundDelivery - await authority.claimWindow(delivery, now) + const delivery = parseStrictInboundDeliveryDetailJson(row.detail_json) + if ( + !delivery || + delivery.userId !== input.userId || + delivery.provider !== mailboxInboundProvider || + delivery.state !== 'pending' || + row.id !== mailboxInboundDedupePointerId(delivery.fingerprint) + ) { + console.warn( + 'inbound-email-dedupe-bridge-row-invalid', + input.userId, + row.id, + ) + continue + } + await authority.claimWindow(delivery, now).catch((error: unknown) => { + console.warn( + 'inbound-email-dedupe-bridge-failed', + input.userId, + row.id, + error, + ) + }) } return await authority.pruneExpiredDedupe(now, input.limit) } diff --git a/packages/worker/src/email/inbound-delivery.ts b/packages/worker/src/email/inbound-delivery.ts index 44726389c7..b5ccc86665 100644 --- a/packages/worker/src/email/inbound-delivery.ts +++ b/packages/worker/src/email/inbound-delivery.ts @@ -10,6 +10,7 @@ import { type UserMeterEnv, } from '#worker/entitlements/user-meter-client.ts' import { normalizeEmailAddress } from './address.ts' +import { systemEmailOwnerId } from './email-owner.ts' import { emailRawMimeKey, getEmailMessageById, @@ -149,12 +150,12 @@ export async function buildInboundDelivery(input: { } } -function parseInboundDelivery( - row: InboundDeliveryEventRow | null, +export function parseInboundDeliveryDetailJson( + detailJson: unknown, ): InboundDelivery | null { - if (!row) return null + if (typeof detailJson !== 'string') return null try { - const detail = JSON.parse(row.detail_json) as Partial + const detail = JSON.parse(detailJson) as Partial if ( typeof detail.deliveryId !== 'string' || typeof detail.fingerprint !== 'string' || @@ -282,6 +283,33 @@ function parseInboundDelivery( } } +export function parseStrictInboundDeliveryDetailJson( + detailJson: unknown, +): InboundDelivery | null { + const delivery = parseInboundDeliveryDetailJson(detailJson) + if (!delivery || typeof detailJson !== 'string') return null + try { + const raw = JSON.parse(detailJson) as unknown + if ( + typeof raw !== 'object' || + raw == null || + !('state' in raw) || + raw.state !== delivery.state + ) { + return null + } + return delivery + } catch { + return null + } +} + +function parseInboundDelivery( + row: InboundDeliveryEventRow | null, +): InboundDelivery | null { + return parseInboundDeliveryDetailJson(row?.detail_json) +} + export async function getInboundDelivery(input: { db: D1Database userId: string @@ -607,6 +635,9 @@ export async function chargeSystemInboundDeliveryOnce(input: { limit: number now: Date }) { + if (input.delivery.userId !== systemEmailOwnerId) { + throw new Error('System inbound delivery charge requires system:email.') + } const existing = await resolveConcurrentDelivery(input) if (existing) return { delivery: existing, overLimit: false as const } const current = await readSystemEmailDailyCounter({ diff --git a/packages/worker/src/email/inbound-effects.ts b/packages/worker/src/email/inbound-effects.ts index cc4f15e98f..ccf0d66943 100644 --- a/packages/worker/src/email/inbound-effects.ts +++ b/packages/worker/src/email/inbound-effects.ts @@ -740,10 +740,22 @@ export async function reconcileInboundDeliveryEffectsForUser(input: { ) .bind(input.userId, input.limit ?? 20) .all<{ id: string }>() - for (const row of bridgeRows.results ?? []) await authority.get(row.id) - const due = await authority.listDueEffects(now, input.limit) let processed = 0 let errors = 0 + for (const row of bridgeRows.results ?? []) { + try { + await authority.get(row.id) + } catch (error) { + errors += 1 + console.warn( + 'inbound-email-effect-bridge-failed', + input.userId, + row.id, + error, + ) + } + } + const due = await authority.listDueEffects(now, input.limit) for (const delivery of due.deliveries) { try { await processInboundDeliveryEffects({ diff --git a/packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts b/packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts index 5281c161a3..bef0e65954 100644 --- a/packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts +++ b/packages/worker/src/email/inbound-mailbox-authority-mirror-migration.node.test.ts @@ -121,6 +121,19 @@ test('0129 promotes legacy inbound authority and cascades usage idempotency', () null, createdAt, ) + insert.run( + 'delivery-system', + 'system:email', + 'received', + 'kody-system-email', + JSON.stringify({ + state: 'received', + fingerprint: 'system-fingerprint', + }), + 0, + null, + createdAt, + ) db.exec('PRAGMA foreign_keys = ON') applyMigrationLikeD1(db, authorityMirrorMigration) @@ -169,6 +182,19 @@ test('0129 promotes legacy inbound authority and cascades usage idempotency', () subscription_effect_last_error: 'dispatch failed', updated_at: createdAt, }) + expect( + db + .prepare( + `SELECT state, fingerprint, updated_at + FROM email_delivery_events + WHERE id = 'delivery-system'`, + ) + .get(), + ).toEqual({ + state: null, + fingerprint: null, + updated_at: createdAt, + }) expect( db .prepare( @@ -191,7 +217,8 @@ test('0129 promotes legacy inbound authority and cascades usage idempotency', () WHERE type = 'index' AND name IN ( 'idx_email_delivery_events_user_state_created', - 'idx_email_delivery_events_user_dedupe_expires' + 'idx_email_delivery_events_user_dedupe_expires', + 'idx_email_inbound_usage_effects_delivery' ) ORDER BY name`, ) @@ -199,6 +226,7 @@ test('0129 promotes legacy inbound authority and cascades usage idempotency', () ).toEqual([ { name: 'idx_email_delivery_events_user_dedupe_expires' }, { name: 'idx_email_delivery_events_user_state_created' }, + { name: 'idx_email_inbound_usage_effects_delivery' }, ]) expect( db diff --git a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts index 6d02302908..2de20c4169 100644 --- a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts +++ b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts @@ -528,8 +528,8 @@ test( handleInboundEmail(first, failingEnv, failCtx.ctx), ).rejects.toBeInstanceOf(RetryableInboundStorageError) expect(first.rejectedReason).toBeNull() - // Charge may schedule UserMeter D1 mirror via waitUntil; no Mailbox - // terminal work should have produced a message below. + // The charged Mailbox delivery survives this retryable graph-storage + // failure; no terminal work should have produced a message below. await drainWaitUntil(failCtx.waitUntilPromises) expect(await readUserDailyReceiveCount(userId)).toBe(1) expect( diff --git a/packages/worker/src/email/inbound-mailbox.node.test.ts b/packages/worker/src/email/inbound-mailbox.node.test.ts index ba262dba89..5bb8142742 100644 --- a/packages/worker/src/email/inbound-mailbox.node.test.ts +++ b/packages/worker/src/email/inbound-mailbox.node.test.ts @@ -10,6 +10,7 @@ const mocks = vi.hoisted(() => ({ eventsTruncated: false, })), authorityGet: vi.fn(async () => ({ state: 'received' as const })), + createAuthority: vi.fn(), processInboundDeliveryEffects: vi.fn(async () => ({ outcome: 'complete' as const, })), @@ -20,7 +21,7 @@ vi.mock('./mailbox-live-mirror.ts', () => ({ })) vi.mock('./inbound-delivery-authority.ts', () => ({ - createUserInboundDeliveryAuthority: () => ({ get: mocks.authorityGet }), + createUserInboundDeliveryAuthority: mocks.createAuthority, })) vi.mock('./inbound-effects.ts', () => ({ @@ -46,6 +47,7 @@ function createCapturedWaitUntilContext() { function resetMocks() { mocks.mirrorMailboxMessageGraphFromD1.mockReset() mocks.authorityGet.mockReset() + mocks.createAuthority.mockReset() mocks.processInboundDeliveryEffects.mockReset() mocks.mirrorMailboxMessageGraphFromD1.mockResolvedValue({ messageId: 'msg-1', @@ -54,6 +56,7 @@ function resetMocks() { eventsTruncated: false, }) mocks.authorityGet.mockResolvedValue({ state: 'received' as const }) + mocks.createAuthority.mockReturnValue({ get: mocks.authorityGet }) mocks.processInboundDeliveryEffects.mockResolvedValue({ outcome: 'complete' as const, }) @@ -222,3 +225,25 @@ test('rejected terminal contains and logs projection repair failures', async () expect.objectContaining({ message: 'projection unavailable' }), ) }) + +test('rejected terminal contains synchronous authority construction failures', async () => { + resetMocks() + consoleError.mockImplementation(() => {}) + mocks.createAuthority.mockImplementation(() => { + throw new Error('MAILBOX binding unavailable') + }) + + await expect( + scheduleInboundRejectedTerminalWork({ + env: { APP_DB: {} } as unknown as Parameters< + typeof scheduleInboundRejectedTerminalWork + >[0]['env'], + userId: 'user-sync-construction', + deliveryId: 'delivery-sync-construction', + }), + ).resolves.toBeUndefined() + expect(consoleError).toHaveBeenCalledWith( + 'Inbound email rejection projection repair failed', + expect.objectContaining({ message: 'MAILBOX binding unavailable' }), + ) +}) diff --git a/packages/worker/src/email/inbound-mailbox.ts b/packages/worker/src/email/inbound-mailbox.ts index 4c84f63a59..6e82151f83 100644 --- a/packages/worker/src/email/inbound-mailbox.ts +++ b/packages/worker/src/email/inbound-mailbox.ts @@ -105,11 +105,14 @@ export async function scheduleInboundRejectedTerminalWork( ) { if (isSystemEmailOwner(input.userId)) return - const task = createUserInboundDeliveryAuthority({ - env: input.env, - userId: input.userId, - }) - .get(input.deliveryId) + const task = Promise.resolve() + .then(async () => { + const authority = createUserInboundDeliveryAuthority({ + env: input.env, + userId: input.userId, + }) + await authority.get(input.deliveryId) + }) .catch((error: unknown) => { console.error('Inbound email rejection projection repair failed', error) }) diff --git a/packages/worker/src/email/inbound.ts b/packages/worker/src/email/inbound.ts index b61780ca44..8926454fef 100644 --- a/packages/worker/src/email/inbound.ts +++ b/packages/worker/src/email/inbound.ts @@ -26,21 +26,23 @@ import { } from './parser.ts' import { buildInboundDelivery, - claimInboundDeliveryWindow, - claimInboundDeliveryStorage, chargeSystemInboundDeliveryOnce, getInboundDeliveryWindow, getInboundDelivery, - markInboundDeliveryRejected, - pruneExpiredInboundDedupePointers, - reconcileStaleInboundDeliveries, readSystemInboundReceiveCount, readUserInboundReceiveCount, - releaseInboundDeliveryStorage, systemInboundQuotaDay, type InboundDelivery, userInboundQuotaDay, } from './inbound-delivery.ts' +import { + claimSystemInboundDeliveryStorage, + claimSystemInboundDeliveryWindow, + markSystemInboundDeliveryRejected, + pruneSystemExpiredInboundDedupePointers, + reconcileSystemStaleInboundDeliveries, + releaseSystemInboundDeliveryStorage, +} from './system-inbound-delivery-authority.ts' import { createUserInboundDeliveryAuthority, type UserInboundDeliveryAuthority, @@ -98,7 +100,7 @@ async function rejectClaimedInboundDelivery(input: { const transitioned = await ( input.authority ? input.authority.reject(input.delivery, input.reason) - : markInboundDeliveryRejected({ + : markSystemInboundDeliveryRejected({ db: input.db, delivery: input.delivery, reason: input.reason, @@ -161,12 +163,12 @@ async function cleanupInboundDurability(input: { }) { try { if (input.userId === systemEmailOwnerId) { - await reconcileStaleInboundDeliveries({ + await reconcileSystemStaleInboundDeliveries({ db: input.env.APP_DB, blobs: input.env.EMAIL_BLOBS, userId: input.userId, }) - await pruneExpiredInboundDedupePointers({ + await pruneSystemExpiredInboundDedupePointers({ db: input.env.APP_DB, userId: input.userId, }) @@ -561,7 +563,7 @@ export async function handleInboundEmail( ...delivery, quotaDay: userInboundQuotaDay(quotaNow), } - claimedDelivery = await authority.charge({ + const chargeResult = await authority.charge({ delivery: chargeCandidate, plan: account.plan, limit: resolveEmailResourceLimit( @@ -570,10 +572,8 @@ export async function handleInboundEmail( ), now: quotaNow, }) - // The charge helper returns the candidate object only when this - // invocation won the atomic D1 charge. A concurrently committed - // delivery is returned as a separately parsed object. - chargedReceive = claimedDelivery === chargeCandidate + claimedDelivery = chargeResult.delivery + chargedReceive = chargeResult.charged } } catch (error) { if (!isEntitlementLimitError(error)) throw error @@ -865,7 +865,7 @@ async function handleSystemInboundEmail(input: { } } if (!activeWindow) { - delivery = await claimInboundDeliveryWindow({ + delivery = await claimSystemInboundDeliveryWindow({ db: input.env.APP_DB, delivery: candidateDelivery, now: quotaNow, @@ -947,7 +947,7 @@ async function handleSystemInboundEmail(input: { await recordReceiveUsage({ outcome: 'error' }) return } - const storageClaim = await claimInboundDeliveryStorage({ + const storageClaim = await claimSystemInboundDeliveryStorage({ db: input.env.APP_DB, delivery: claimedDelivery, expectedAttachmentCount: parsed.attachments.length, @@ -968,7 +968,7 @@ async function handleSystemInboundEmail(input: { parsed, }) } catch (error) { - await releaseInboundDeliveryStorage({ + await releaseSystemInboundDeliveryStorage({ db: input.env.APP_DB, delivery: storageClaim.delivery, }).catch((releaseError) => { diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 1fd38bffb9..e8f9ec25a2 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -26,6 +26,10 @@ import { markMailboxInboundDeliveryOrphanCleaned, releaseMailboxInboundDeliveryCleanup, } from './mailbox-inbound-cleanup-ledger.ts' +import { + assertMailboxD1DeliveryEventBatch, + shouldSkipMailboxDeliveryEventWrite, +} from './mailbox-inbound-bootstrap.ts' import { claimMailboxInboundDeliveryStorage, claimMailboxInboundDeliveryWindow, @@ -95,7 +99,6 @@ import { class MailboxBase extends DurableObject implements MailboxRpc { private readonly store: MailboxStore - /** In-isolate alarm scheduling caches. */ private retentionAlarmArmed = false private retentionIdleConfirmed = false @@ -212,7 +215,6 @@ class MailboxBase extends DurableObject implements MailboxRpc { return this.runRetentionPass() } - /** Atomic mirror of thread + message + attachments for dual-write. */ async mirrorMessage(input: { ownerId: string thread?: MailboxThreadInput | null @@ -243,13 +245,10 @@ class MailboxBase extends DurableObject implements MailboxRpc { return { ok: true, accepted } } - /** - * Upsert a delivery event (idempotent on `provider_event_id`) and optionally - * apply a monotonic latest `delivery_status` update on the message. - */ async upsertDeliveryEvent(input: { ownerId: string event: MailboxDeliveryEventInput + intent?: 'user-inbound-bootstrap' latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus @@ -265,6 +264,16 @@ class MailboxBase extends DurableObject implements MailboxRpc { let updatedLatestStatus = false this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) + if ( + shouldSkipMailboxDeliveryEventWrite(this.ctx.storage.sql, { + ownerId: input.ownerId, + event: input.event, + intent: input.intent, + hasLatestDeliveryStatus: input.latestDeliveryStatus != null, + }) + ) { + return + } const write = this.store.writeDeliveryEventRow(input.event) inserted = write.inserted accepted = write.accepted @@ -286,12 +295,6 @@ class MailboxBase extends DurableObject implements MailboxRpc { return { inserted, accepted, updatedLatestStatus } } - /** - * Upsert a bounded batch of complete immutable delivery-event snapshots in - * one transaction / one DO RPC. Validates non-empty and - * {@link mailboxUpsertDeliveryEventsMax}. Does not patch message latest - * delivery status. Marks retention dirty once for the batch. - */ async upsertDeliveryEvents(input: { ownerId: string events: Array @@ -304,6 +307,7 @@ class MailboxBase extends DurableObject implements MailboxRpc { `Mailbox upsertDeliveryEvents events exceed max of ${mailboxUpsertDeliveryEventsMax}.`, ) } + assertMailboxD1DeliveryEventBatch(input.events) const results: Array = [] this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) @@ -682,6 +686,9 @@ class MailboxBase extends DurableObject implements MailboxRpc { async claimInboundDeliveryCleanup(input: { ownerId: string deliveryId: string + expectedState: MailboxInboundDeliveryState + expectedUpdatedAt: string + staleBefore: string now?: string }) { let result!: Awaited> @@ -838,10 +845,6 @@ class MailboxBase extends DurableObject implements MailboxRpc { return listMailboxDueInboundEffectWork(this.ctx.storage.sql, input) } - /** - * Clear SQLite state only. During expand, D1 deletion remains authoritative - * for R2 objects. - */ async purge(): Promise<{ ok: true }> { await this.ctx.blockConcurrencyWhile(async () => { await this.ctx.storage.deleteAlarm().catch(() => { diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index a753e9ae4b..3d83108a9f 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -496,8 +496,24 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso updatedLatestStatus: true, }) + const inboundDetail = { + fingerprint: 'fp-abc', + deliveryId: 'inbound-delivery-1', + messageId: 'inbound-msg-1', + threadId: 'inbound-thread-1', + rawMimeKey: emailRawMimeKey(userId, 'inbound-msg-1'), + userId, + inboxId: 'inbox-1', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + provider: 'cloudflare-email-routing', + quotaDay: '2026-07-02', + dedupeExpiresAt: '2026-07-04T11:00:00.000Z', + state: 'received' as const, + } const inbound = await mailbox.upsertDeliveryEvent({ ownerId: userId, + intent: 'user-inbound-bootstrap', event: baseDeliveryEvent({ id: 'inbound-delivery-1', messageId: 'inbound-msg-1', @@ -517,7 +533,7 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso usageDurationMs: 120, subscriptionEffectState: 'pending', subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', - detailJson: JSON.stringify({ recipient: 'owner@example.com' }), + detailJson: JSON.stringify(inboundDetail), }), }) expect(inbound.inserted).toBe(true) @@ -544,6 +560,28 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso expect(JSON.parse(inboundRow!.detailJson)).toMatchObject({ recipient: 'owner@example.com', }) + const repeatedBootstrap = await mailbox.upsertDeliveryEvent({ + ownerId: userId, + intent: 'user-inbound-bootstrap', + event: baseDeliveryEvent({ + id: 'inbound-delivery-1', + messageId: 'inbound-msg-1', + inboxId: 'inbox-1', + eventType: 'rejected', + provider: 'cloudflare-email-routing', + createdAt: '2026-07-02T11:00:00.000Z', + updatedAt: '2026-07-02T12:00:00.000Z', + needsEffectReconcile: false, + state: 'rejected', + fingerprint: 'fp-abc', + detailJson: JSON.stringify({ ...inboundDetail, state: 'rejected' }), + }), + }) + expect(repeatedBootstrap).toEqual({ + inserted: false, + accepted: false, + updatedLatestStatus: false, + }) // Explicit needsEffectReconcile: false is stored as false. await mailbox.upsertDeliveryEvent({ diff --git a/packages/worker/src/email/mailbox-inbound-bootstrap.ts b/packages/worker/src/email/mailbox-inbound-bootstrap.ts new file mode 100644 index 0000000000..5b5e5974ea --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-bootstrap.ts @@ -0,0 +1,85 @@ +import { + mailboxInboundDedupePointerId, + mailboxInboundDedupeProvider, + mailboxInboundProvider, +} from './mailbox-inbound-ledger.ts' +import { parseStrictInboundDeliveryDetailJson } from './inbound-delivery.ts' +import { + assertMailboxNonEmptyString, + type MailboxDeliveryEventInput, +} from './mailbox-types.ts' + +function isUserInboundAuthorityEvent(event: MailboxDeliveryEventInput) { + return ( + event.provider === mailboxInboundProvider || + event.provider === mailboxInboundDedupeProvider + ) +} + +/** + * Fence the only D1 → Mailbox USER inbound bridge. Bootstrap accepts a + * validated complete snapshot only when the owner-bound row is still missing. + */ +export function shouldSkipMailboxDeliveryEventWrite( + sql: SqlStorage, + input: { + ownerId: string + event: MailboxDeliveryEventInput + intent?: 'user-inbound-bootstrap' + hasLatestDeliveryStatus: boolean + }, +) { + const inboundAuthorityEvent = isUserInboundAuthorityEvent(input.event) + if (inboundAuthorityEvent && input.intent !== 'user-inbound-bootstrap') { + throw new Error( + 'USER inbound delivery events require the missing-row bootstrap intent.', + ) + } + if (!inboundAuthorityEvent && input.intent === 'user-inbound-bootstrap') { + throw new Error( + 'USER inbound bootstrap intent requires an inbound delivery provider.', + ) + } + if (input.intent !== 'user-inbound-bootstrap') return false + if (input.hasLatestDeliveryStatus) { + throw new Error( + 'USER inbound bootstrap cannot update latest delivery status.', + ) + } + const delivery = parseStrictInboundDeliveryDetailJson(input.event.detailJson) + const expectedEventId = + input.event.provider === mailboxInboundDedupeProvider && delivery + ? mailboxInboundDedupePointerId(delivery.fingerprint) + : delivery?.deliveryId + if ( + !delivery || + delivery.userId !== input.ownerId || + delivery.provider !== mailboxInboundProvider || + input.event.id !== expectedEventId || + input.event.state !== delivery.state || + input.event.fingerprint !== delivery.fingerprint + ) { + throw new Error( + 'USER inbound bootstrap requires a valid owner-bound complete snapshot.', + ) + } + const eventId = assertMailboxNonEmptyString(input.event.id, 'event.id') + return ( + sql + .exec( + `SELECT id FROM email_delivery_events WHERE id = ? LIMIT 1`, + eventId, + ) + .toArray().length > 0 + ) +} + +export function assertMailboxD1DeliveryEventBatch( + events: Array, +) { + if (events.some(isUserInboundAuthorityEvent)) { + throw new Error( + 'USER inbound delivery events cannot use the D1-authoritative batch mirror.', + ) + } +} diff --git a/packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts b/packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts index dd172c1e07..0e5d72f208 100644 --- a/packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-cleanup-ledger.ts @@ -1,4 +1,9 @@ -import { assertMailboxNonEmptyString } from './mailbox-types.ts' +import { + assertMailboxCanonicalIsoTimestamp, + assertMailboxInboundDeliveryState, + assertMailboxNonEmptyString, + type MailboxInboundDeliveryState, +} from './mailbox-types.ts' import { detailJsonFromMailboxInboundSnapshot, mailboxInboundProvider, @@ -15,26 +20,45 @@ import { mailboxInboundOrphanVerificationMs } from './mailbox-inbound-ledger-sha export function claimMailboxInboundDeliveryCleanup( sql: SqlStorage, - input: { deliveryId: string; now?: string }, + input: { + deliveryId: string + expectedState: MailboxInboundDeliveryState + expectedUpdatedAt: string + staleBefore: string + now?: string + }, ): MailboxClaimInboundDeliveryCleanupResult { const now = normalizeMailboxInboundNow(input.now) const deliveryId = assertMailboxNonEmptyString(input.deliveryId, 'deliveryId') + const expectedState = assertMailboxInboundDeliveryState(input.expectedState) + const expectedUpdatedAt = assertMailboxCanonicalIsoTimestamp( + input.expectedUpdatedAt, + 'expectedUpdatedAt', + ) + const staleBefore = assertMailboxCanonicalIsoTimestamp( + input.staleBefore, + 'staleBefore', + ) const current = readMailboxInboundDeliveryById(sql, deliveryId) if (!current) return { status: 'not-claimed', delivery: null } const leaseExpiredBefore = new Date( Date.parse(now) - mailboxInboundStorageLeaseMs, ).toISOString() const claimable = - current.state === 'pending' || - (current.state === 'storing' && - current.storageLeaseAt != null && - current.storageLeaseAt < leaseExpiredBefore) || - (current.state === 'cleaning' && - current.cleanupLeaseAt != null && - current.cleanupLeaseAt < leaseExpiredBefore) || - (current.state === 'orphan-cleaned' && - current.cleanupRetryAt != null && - current.cleanupRetryAt <= now) + current.state === expectedState && + current.updatedAt === expectedUpdatedAt && + current.createdAt < staleBefore && + (current.reconcileAfter == null || current.reconcileAfter <= now) && + (current.state === 'pending' || + (current.state === 'storing' && + current.storageLeaseAt != null && + current.storageLeaseAt < leaseExpiredBefore) || + (current.state === 'cleaning' && + current.cleanupLeaseAt != null && + current.cleanupLeaseAt < leaseExpiredBefore) || + (current.state === 'orphan-cleaned' && + current.cleanupRetryAt != null && + current.cleanupRetryAt <= now)) if (!claimable) return { status: 'not-claimed', delivery: current } const cleanupLease = crypto.randomUUID() @@ -54,6 +78,10 @@ export function claimMailboxInboundDeliveryCleanup( updated_at = ? WHERE id = ? AND provider = ? + AND state = ? + AND updated_at = ? + AND created_at < ? + AND (reconcile_after IS NULL OR reconcile_after <= ?) AND ( state = 'pending' OR (state = 'storing' AND storage_lease_at < ?) @@ -66,6 +94,10 @@ export function claimMailboxInboundDeliveryCleanup( now, deliveryId, mailboxInboundProvider, + expectedState, + expectedUpdatedAt, + staleBefore, + now, leaseExpiredBefore, leaseExpiredBefore, now, diff --git a/packages/worker/src/email/mailbox-inbound-ledger.ts b/packages/worker/src/email/mailbox-inbound-ledger.ts index 47e7c8d277..d0f971865c 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.ts @@ -1,8 +1,8 @@ /** * Mailbox USER inbound ledger authority CAS primitives. * - * Owner-bound SQLite helpers matching D1 USER transitions in - * `inbound-delivery.ts`. Effect lease CAS lives in + * Owner-bound SQLite helpers implementing authoritative USER transitions. + * The legacy D1 implementation is system-only. Effect lease CAS lives in * `mailbox-inbound-effect-ledger.ts`. No external usage/subscription side * effects. D1 receives synchronous compatibility snapshots after CAS; * `system:email` stays on D1. @@ -186,6 +186,9 @@ export type MailboxInboundDeliveryLedgerRpc = { claimInboundDeliveryCleanup: (input: { ownerId: string deliveryId: string + expectedState: MailboxInboundDeliveryState + expectedUpdatedAt: string + staleBefore: string now?: string }) => Promise releaseInboundDeliveryCleanup: (input: { diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts index bf94d26d8b..7b30dc43a0 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -7,8 +7,10 @@ import { insertInput } from './mailbox-inbound-ledger-test-helpers.ts' import { mailboxInboundDedupeProvider, mailboxInboundProvider, + mailboxInboundReconciliationRetryMs, mailboxInboundStorageLeaseMs, } from './mailbox-inbound-ledger.ts' +import { mailboxInboundOrphanVerificationMs } from './mailbox-inbound-ledger-shared.ts' import { initializeMailboxSchema } from './mailbox-schema.ts' import { mailboxMetaSchemaVersionKey, @@ -252,9 +254,60 @@ test('Mailbox inbound ledger CAS covers USER authority transition matrix', async expect( dueStale.deliveries.some((d) => d.deliveryId === stalePending.deliveryId), ).toBe(true) + const racedPending = insertInput(ownerA, { + fingerprint: 'fp-stale-race', + deliveryId: 'email-inbound-delivery:stale-race', + messageId: 'email-inbound-message:stale-race', + threadId: 'email-inbound-thread:stale-race', + }) + await mailboxA.insertChargedPendingInboundDelivery({ + ownerId: ownerA, + delivery: racedPending, + now: '2026-07-19T00:00:00.000Z', + }) + const racedDue = await mailboxA.listDueStaleInboundDeliveries({ + ownerId: ownerA, + now, + limit: 50, + }) + const racedSnapshot = racedDue.deliveries.find( + (delivery) => delivery.deliveryId === racedPending.deliveryId, + ) + if (!racedSnapshot) throw new Error('expected stale race snapshot') + const racedStorage = await mailboxA.claimInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: racedPending.deliveryId, + expectedAttachmentCount: 0, + now: '2026-07-22T00:00:01.000Z', + }) + if ( + racedStorage.status !== 'claimed' || + !racedStorage.delivery.storageLease + ) { + throw new Error('expected raced storage claim') + } + await mailboxA.releaseInboundDeliveryStorage({ + ownerId: ownerA, + deliveryId: racedPending.deliveryId, + storageLease: racedStorage.delivery.storageLease, + now: '2026-07-22T00:00:02.000Z', + }) + expect( + await mailboxA.claimInboundDeliveryCleanup({ + ownerId: ownerA, + deliveryId: racedPending.deliveryId, + expectedState: racedSnapshot.state, + expectedUpdatedAt: racedSnapshot.updatedAt, + staleBefore: '2026-07-21T00:00:00.000Z', + now: '2026-07-22T00:00:03.000Z', + }), + ).toMatchObject({ status: 'not-claimed' }) const cleanupClaim = await mailboxA.claimInboundDeliveryCleanup({ ownerId: ownerA, deliveryId: stalePending.deliveryId, + expectedState: 'pending', + expectedUpdatedAt: '2026-07-19T00:00:00.000Z', + staleBefore: '2026-07-21T00:00:00.000Z', now, }) expect(cleanupClaim.status).toBe('claimed') @@ -324,10 +377,14 @@ test('Mailbox inbound ledger CAS covers USER authority transition matrix', async now, }) expect(deferred.status).toBe('deferred') + const orphanNow = '2026-07-22T00:30:00.000Z' const orphanClaim = await mailboxA.claimInboundDeliveryCleanup({ ownerId: ownerA, deliveryId: stalePending.deliveryId, - now, + expectedState: 'pending', + expectedUpdatedAt: now, + staleBefore: '2026-07-21T00:00:00.000Z', + now: orphanNow, }) if (orphanClaim.status !== 'claimed' || !orphanClaim.delivery.cleanupLease) { throw new Error('expected orphan cleanup claim') @@ -338,7 +395,7 @@ test('Mailbox inbound ledger CAS covers USER authority transition matrix', async deliveryId: stalePending.deliveryId, cleanupLease: 'stale-cleanup-lease', outcome: 'deleted', - now, + now: orphanNow, }), ).toEqual({ status: 'lease-lost' }) const orphaned = await mailboxA.markInboundDeliveryOrphanCleaned({ @@ -346,9 +403,48 @@ test('Mailbox inbound ledger CAS covers USER authority transition matrix', async deliveryId: stalePending.deliveryId, cleanupLease: orphanClaim.delivery.cleanupLease, outcome: 'delete-failed', - now, + now: orphanNow, }) expect(orphaned.status).toBe('orphan-cleaned') + if (orphaned.status !== 'orphan-cleaned') { + throw new Error('expected orphan-cleaned result') + } + expect(orphaned.delivery.cleanupRetryAt).toBe( + new Date( + Date.parse(orphanNow) + mailboxInboundReconciliationRetryMs, + ).toISOString(), + ) + const deletedNow = '2026-07-22T01:00:00.000Z' + const deletedClaim = await mailboxA.claimInboundDeliveryCleanup({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + expectedState: 'orphan-cleaned', + expectedUpdatedAt: orphanNow, + staleBefore: '2026-07-21T00:00:00.000Z', + now: deletedNow, + }) + if ( + deletedClaim.status !== 'claimed' || + !deletedClaim.delivery.cleanupLease + ) { + throw new Error('expected deleted verification cleanup claim') + } + const deleted = await mailboxA.markInboundDeliveryOrphanCleaned({ + ownerId: ownerA, + deliveryId: stalePending.deliveryId, + cleanupLease: deletedClaim.delivery.cleanupLease, + outcome: 'deleted', + now: deletedNow, + }) + expect(deleted.status).toBe('orphan-cleaned') + if (deleted.status !== 'orphan-cleaned') { + throw new Error('expected deleted orphan-cleaned result') + } + expect(deleted.delivery.cleanupRetryAt).toBe( + new Date( + Date.parse(deletedNow) + mailboxInboundOrphanVerificationMs, + ).toISOString(), + ) // Mirror upsert compatibility still works alongside ledger rows. const mirror = await mailboxA.upsertDeliveryEvent({ diff --git a/packages/worker/src/email/mailbox-live-mirror.ts b/packages/worker/src/email/mailbox-live-mirror.ts index e0252baf4c..83f609621c 100644 --- a/packages/worker/src/email/mailbox-live-mirror.ts +++ b/packages/worker/src/email/mailbox-live-mirror.ts @@ -17,6 +17,10 @@ import { listEmailAttachmentsForMessage, } from './repo.ts' import { type EmailThreadRecord } from './types.ts' +import { + mailboxInboundDedupeProvider, + mailboxInboundProvider, +} from './mailbox-inbound-ledger.ts' /** * High-level D1 → Mailbox live-mirror orchestration. @@ -46,6 +50,13 @@ export const mailboxLiveMirrorMaxAnalyticsWrites = 2 export type MailboxLiveMirrorEnv = MailboxMirrorEnv +function isUserInboundAuthorityProvider(provider: string | null) { + return ( + provider === mailboxInboundProvider || + provider === mailboxInboundDedupeProvider + ) +} + export type MailboxLiveMirrorEventResult = { eventId: string result: MailboxMirrorResult @@ -96,6 +107,9 @@ export async function mirrorMailboxDeliveryEventFromD1(input: { eventId: input.eventId, }) if (!projection) return { status: 'missing' } + if (isUserInboundAuthorityProvider(projection.provider)) { + return { status: 'skipped', reason: 'user-inbound-authority' } + } const sourceMutationAt = input.sourceMutationAt != null && input.sourceMutationAt.length > 0 ? input.sourceMutationAt @@ -193,9 +207,11 @@ export async function mirrorMailboxMessageGraphFromD1(input: { max: mailboxLiveMirrorMaxEvents, }) } - const eventInputs = eventsTruncated - ? loadedEvents.slice(-mailboxLiveMirrorMaxEvents) - : loadedEvents + const eventInputs = ( + eventsTruncated + ? loadedEvents.slice(-mailboxLiveMirrorMaxEvents) + : loadedEvents + ).filter((event) => !isUserInboundAuthorityProvider(event.provider)) const events = eventInputs.length === 0 diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index a5dbd44afa..d6329e12a1 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -62,6 +62,7 @@ export type MailboxMirrorSkipReason = | 'system-email' | 'mailbox-unconfigured' | 'missing-owner' + | 'user-inbound-authority' export type MailboxMirrorResult = | { status: 'mirrored' } diff --git a/packages/worker/src/email/mailbox-snapshot-repo.ts b/packages/worker/src/email/mailbox-snapshot-repo.ts index cd7ea51392..30f2823b20 100644 --- a/packages/worker/src/email/mailbox-snapshot-repo.ts +++ b/packages/worker/src/email/mailbox-snapshot-repo.ts @@ -105,9 +105,8 @@ export async function getEmailDeliveryEventMirrorProjection(input: { /** * Load and convert one ready Mailbox delivery-event snapshot for dual-write. * - * @param sourceMutationAt - Canonical mirror `updatedAt`. D1 delivery events - * lack `updated_at`. Phase-2 high-risk lifecycle mutations must pass the - * same timestamp used for the D1 mutation; inserts use `created_at`. + * @param sourceMutationAt - Canonical mirror `updatedAt` for D1-authoritative + * message-graph and non-USER-inbound events; inserts use `created_at`. */ export async function getMailboxDeliveryEventMirrorInput(input: { db: D1Database diff --git a/packages/worker/src/email/mailbox-snapshots.ts b/packages/worker/src/email/mailbox-snapshots.ts index 20d2c5a6c3..16769acda2 100644 --- a/packages/worker/src/email/mailbox-snapshots.ts +++ b/packages/worker/src/email/mailbox-snapshots.ts @@ -33,9 +33,8 @@ import { */ /** - * Complete D1 delivery-event mirror projection: base row fields plus - * authoritative promoted columns. `detail_json` still owns inbound/effect - * lease fields that have not been promoted. + * Complete D1 delivery-event mirror projection for D1-authoritative events: + * base row fields plus promoted compatibility columns. * * Authoritative D1 columns (never read from `detail_json`): * - `needs_effect_reconcile` @@ -214,9 +213,7 @@ export function toMailboxAttachmentInput( * Convert a complete D1 delivery-event projection into a Mailbox input. * Promoted columns win; remaining inbound/effect fields come from JSON. * - * `sourceMutationAt` becomes Mailbox `updatedAt`. D1 delivery events lack - * `updated_at`: phase-2 high-risk lifecycle mutations must pass the same - * canonical timestamp used for the D1 mutation; inserts use `created_at`. + * `sourceMutationAt` becomes Mailbox `updatedAt`; inserts use `created_at`. */ export function toMailboxDeliveryEventInput(input: { projection: EmailDeliveryEventMirrorProjection diff --git a/packages/worker/src/email/mailbox-store.ts b/packages/worker/src/email/mailbox-store.ts index 62964e5f46..6d989e07b7 100644 --- a/packages/worker/src/email/mailbox-store.ts +++ b/packages/worker/src/email/mailbox-store.ts @@ -106,10 +106,10 @@ function buildMailboxMessageFilterClauses(input: { /** * SQLite write/query helpers for one Mailbox DO. No alarm / R2 side effects. * - * Additive step 2a USER inbound ledger/effect CAS helpers live in - * `mailbox-inbound-ledger.ts` / `mailbox-inbound-effect-ledger.ts` (wired as - * owner-bound DO RPCs; not live-called — D1 remains authority). Mirror upsert - * paths below stay the compatibility write API. + * USER inbound ledger/effect CAS helpers live in + * `mailbox-inbound-ledger.ts` / `mailbox-inbound-effect-ledger.ts` and are + * authoritative. Generic mirror upserts are fenced from USER inbound providers; + * only the missing-row bootstrap bridge may insert those snapshots. */ export class MailboxStore { private readonly storage: DurableObjectStorage diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index 98d7ee9498..aa4f4683cc 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -505,6 +505,8 @@ type MailboxCoreRpc = { upsertDeliveryEvent: (input: { ownerId: string event: MailboxDeliveryEventInput + /** Missing-row-only bridge for pre-deploy USER inbound D1 snapshots. */ + intent?: 'user-inbound-bootstrap' latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus diff --git a/packages/worker/src/email/reconcile-inbound-deliveries.ts b/packages/worker/src/email/reconcile-inbound-deliveries.ts index 1498d43cb4..557f171e99 100644 --- a/packages/worker/src/email/reconcile-inbound-deliveries.ts +++ b/packages/worker/src/email/reconcile-inbound-deliveries.ts @@ -1,14 +1,14 @@ -import { - pruneExpiredInboundDedupePointers, - reconcileStaleInboundDeliveries, - staleInboundDeliveryAgeMs, -} from './inbound-delivery.ts' +import { staleInboundDeliveryAgeMs } from './inbound-delivery.ts' import { reconcileInboundDeliveryEffectsForUser } from './inbound-effects.ts' import { pruneUserExpiredInboundDedupePointers, reconcileUserStaleInboundDeliveries, } from './inbound-delivery-reconciliation-authority.ts' import { systemEmailOwnerId } from './email-owner.ts' +import { + pruneSystemExpiredInboundDedupePointers, + reconcileSystemStaleInboundDeliveries, +} from './system-inbound-delivery-authority.ts' import { withAccountWriteLease } from '#worker/account/deletion-state.ts' const reconciliationUserBatchSize = 25 @@ -195,7 +195,7 @@ export async function sweepStaleInboundDeliveries(input: { const reconcileUser = async () => { const systemOwner = userId === systemEmailOwnerId const result = systemOwner - ? await reconcileStaleInboundDeliveries({ + ? await reconcileSystemStaleInboundDeliveries({ db: input.env.APP_DB, blobs: input.env.EMAIL_BLOBS, userId, @@ -209,7 +209,7 @@ export async function sweepStaleInboundDeliveries(input: { deadlineMs, }) const pruned = systemOwner - ? await pruneExpiredInboundDedupePointers({ + ? await pruneSystemExpiredInboundDedupePointers({ db: input.env.APP_DB, userId, now, diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index 97d224fba6..241fab02a4 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -4,12 +4,9 @@ import PostalMime from 'postal-mime' import { withAccountWriteLease } from '#worker/account/deletion-state.ts' import { normalizeEmailAddress } from './address.ts' import { resolveInboundEmailAuthVerdict } from './auth-verdict.ts' -import { - getInboundDelivery, - markInboundDeliveryReceived, - type InboundDelivery, -} from './inbound-delivery.ts' +import { getInboundDelivery, type InboundDelivery } from './inbound-delivery.ts' import { type UserInboundDeliveryAuthority } from './inbound-delivery-authority.ts' +import { markSystemInboundDeliveryReceived } from './system-inbound-delivery-authority.ts' import { mirrorMailboxMessageGraphFromD1, type MailboxLiveMirrorEnv, @@ -508,7 +505,7 @@ export async function storeIdempotentInboundEmail(input: { } finalizedDelivery = input.authority ? await input.authority.receive(finalization) - : await markInboundDeliveryReceived({ + : await markSystemInboundDeliveryReceived({ db: input.db, ...finalization, }) diff --git a/packages/worker/src/email/system-inbound-delivery-authority.ts b/packages/worker/src/email/system-inbound-delivery-authority.ts new file mode 100644 index 0000000000..f833acd3cb --- /dev/null +++ b/packages/worker/src/email/system-inbound-delivery-authority.ts @@ -0,0 +1,72 @@ +import { + claimInboundDeliveryStorage, + claimInboundDeliveryWindow, + markInboundDeliveryReceived, + markInboundDeliveryRejected, + pruneExpiredInboundDedupePointers, + reconcileStaleInboundDeliveries, + releaseInboundDeliveryStorage, + type InboundDelivery, +} from './inbound-delivery.ts' +import { systemEmailOwnerId } from './email-owner.ts' + +function assertSystemInboundOwner(userId: string) { + if (userId !== systemEmailOwnerId) { + throw new Error( + 'Legacy D1 inbound delivery mutations are restricted to system:email.', + ) + } +} + +function assertSystemInboundDelivery(delivery: InboundDelivery) { + assertSystemInboundOwner(delivery.userId) +} + +export async function claimSystemInboundDeliveryWindow( + input: Parameters[0], +) { + assertSystemInboundDelivery(input.delivery) + return await claimInboundDeliveryWindow(input) +} + +export async function claimSystemInboundDeliveryStorage( + input: Parameters[0], +) { + assertSystemInboundDelivery(input.delivery) + return await claimInboundDeliveryStorage(input) +} + +export async function releaseSystemInboundDeliveryStorage( + input: Parameters[0], +) { + assertSystemInboundDelivery(input.delivery) + return await releaseInboundDeliveryStorage(input) +} + +export async function markSystemInboundDeliveryRejected( + input: Parameters[0], +) { + assertSystemInboundDelivery(input.delivery) + return await markInboundDeliveryRejected(input) +} + +export async function markSystemInboundDeliveryReceived( + input: Parameters[0], +) { + assertSystemInboundDelivery(input.delivery) + return await markInboundDeliveryReceived(input) +} + +export async function pruneSystemExpiredInboundDedupePointers( + input: Parameters[0], +) { + assertSystemInboundOwner(input.userId) + return await pruneExpiredInboundDedupePointers(input) +} + +export async function reconcileSystemStaleInboundDeliveries( + input: Parameters[0], +) { + assertSystemInboundOwner(input.userId) + return await reconcileStaleInboundDeliveries(input) +} diff --git a/packages/worker/src/email/test-schema.ts b/packages/worker/src/email/test-schema.ts index 84a2425a1d..787df7fc15 100644 --- a/packages/worker/src/email/test-schema.ts +++ b/packages/worker/src/email/test-schema.ts @@ -194,6 +194,13 @@ ON email_delivery_events(user_id, created_at) WHERE provider = 'cloudflare-email-routing' AND event_type = 'received' AND needs_effect_reconcile = 1;`, + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_user_state_created +ON email_delivery_events(user_id, state, created_at, id) +WHERE provider = 'cloudflare-email-routing' AND state IS NOT NULL;`, + `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_user_dedupe_expires +ON email_delivery_events(user_id, dedupe_expires_at, id) +WHERE provider = 'cloudflare-email-routing-dedupe' + AND dedupe_expires_at IS NOT NULL;`, `CREATE TABLE IF NOT EXISTS email_inbound_usage_effects ( user_id TEXT NOT NULL, delivery_id TEXT NOT NULL, @@ -202,6 +209,8 @@ WHERE provider = 'cloudflare-email-routing' PRIMARY KEY (user_id, delivery_id, finalization_token), FOREIGN KEY (delivery_id) REFERENCES email_delivery_events(id) ON DELETE CASCADE );`, + `CREATE INDEX IF NOT EXISTS idx_email_inbound_usage_effects_delivery +ON email_inbound_usage_effects(delivery_id);`, `CREATE INDEX IF NOT EXISTS idx_email_delivery_events_recorded_usage_month ON email_delivery_events(usage_month, user_id) WHERE provider = 'cloudflare-email-routing' diff --git a/tools/migration-ledger.json b/tools/migration-ledger.json index d871923d4e..487c16e6cd 100644 --- a/tools/migration-ledger.json +++ b/tools/migration-ledger.json @@ -530,7 +530,7 @@ }, { "filename": "0129-email-inbound-mailbox-authority-mirror.sql", - "sha256": "18ba77426f929819989c450ec5e650438a74ce692f3bd73197a46f843412028e" + "sha256": "2b5d01a5b5f2adf7ab145a17d7a2cc58b06398cc08850838ffba537f23a31b59" } ] } From 29e8a6100acd619993f805d4a3c2fc743282dfe6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 17:40:09 +0000 Subject: [PATCH 58/62] fix(email): preserve preclaim parity audits Co-authored-by: Kent C. Dodds --- packages/worker/src/email/mailbox-do.ts | 27 ++-- .../src/email/mailbox-inbound-bootstrap.ts | 131 ++++++++++++++++-- .../email/mailbox-reconcile.workers.test.ts | 35 +++-- 3 files changed, 157 insertions(+), 36 deletions(-) diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index e8f9ec25a2..9ece7acf35 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -26,10 +26,7 @@ import { markMailboxInboundDeliveryOrphanCleaned, releaseMailboxInboundDeliveryCleanup, } from './mailbox-inbound-cleanup-ledger.ts' -import { - assertMailboxD1DeliveryEventBatch, - shouldSkipMailboxDeliveryEventWrite, -} from './mailbox-inbound-bootstrap.ts' +import { shouldSkipMailboxDeliveryEventWrite } from './mailbox-inbound-bootstrap.ts' import { claimMailboxInboundDeliveryStorage, claimMailboxInboundDeliveryWindow, @@ -107,7 +104,6 @@ class MailboxBase extends DurableObject implements MailboxRpc { this.store = new MailboxStore(ctx.storage) this.ctx.blockConcurrencyWhile(async () => { this.store.initializeSchema() - // Observe existing alarm only — never arm in the constructor. this.retentionAlarmArmed = (await this.ctx.storage.getAlarm()) != null }) } @@ -307,18 +303,23 @@ class MailboxBase extends DurableObject implements MailboxRpc { `Mailbox upsertDeliveryEvents events exceed max of ${mailboxUpsertDeliveryEventsMax}.`, ) } - assertMailboxD1DeliveryEventBatch(input.events) const results: Array = [] this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) for (const event of input.events) { const eventId = assertMailboxNonEmptyString(event.id, 'event.id') + if ( + shouldSkipMailboxDeliveryEventWrite(this.ctx.storage.sql, { + ownerId: input.ownerId, + event, + hasLatestDeliveryStatus: false, + }) + ) { + results.push({ eventId, inserted: false, accepted: false }) + continue + } const write = this.store.writeDeliveryEventRow(event) - results.push({ - eventId, - inserted: write.inserted, - accepted: write.accepted, - }) + results.push({ eventId, ...write }) } }) this.markRetentionDirty() @@ -847,9 +848,7 @@ class MailboxBase extends DurableObject implements MailboxRpc { async purge(): Promise<{ ok: true }> { await this.ctx.blockConcurrencyWhile(async () => { - await this.ctx.storage.deleteAlarm().catch(() => { - // Best effort: deleteAll below still clears persisted alarm state. - }) + await this.ctx.storage.deleteAlarm().catch(() => undefined) await this.ctx.storage.deleteAll() this.retentionAlarmArmed = false this.retentionIdleConfirmed = true diff --git a/packages/worker/src/email/mailbox-inbound-bootstrap.ts b/packages/worker/src/email/mailbox-inbound-bootstrap.ts index 5b5e5974ea..18f52bad45 100644 --- a/packages/worker/src/email/mailbox-inbound-bootstrap.ts +++ b/packages/worker/src/email/mailbox-inbound-bootstrap.ts @@ -4,11 +4,28 @@ import { mailboxInboundProvider, } from './mailbox-inbound-ledger.ts' import { parseStrictInboundDeliveryDetailJson } from './inbound-delivery.ts' +import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' import { assertMailboxNonEmptyString, type MailboxDeliveryEventInput, + type MailboxDeliveryEventRecord, } from './mailbox-types.ts' +const preClaimAuditPhases = new Set([ + 'entitlement', + 'size', + 'account-verification', + 'account-suspension', + 'sender-policy', + 'system-limit', +]) +const auditDayPattern = /^\d{4}-\d{2}-\d{2}$/ + +function isCanonicalTimestamp(value: string) { + const parsed = Date.parse(value) + return Number.isFinite(parsed) && new Date(parsed).toISOString() === value +} + function isUserInboundAuthorityEvent(event: MailboxDeliveryEventInput) { return ( event.provider === mailboxInboundProvider || @@ -16,6 +33,100 @@ function isUserInboundAuthorityEvent(event: MailboxDeliveryEventInput) { ) } +function hasNoInboundAuthorityState( + event: MailboxDeliveryEventInput | MailboxDeliveryEventRecord, +) { + return ( + event.messageId == null && + event.providerMessageId == null && + event.providerEventId == null && + event.state == null && + event.fingerprint == null && + event.storageLease == null && + event.storageLeaseAt == null && + event.cleanupLease == null && + event.cleanupLeaseAt == null && + event.cleanupRetryAt == null && + event.expectedAttachmentCount == null && + event.finalizationToken == null && + event.reconcileAfter == null && + event.dedupeExpiresAt == null && + event.usageEffectRecordedAt == null && + event.usageEffectSuppressedAt == null && + event.usageStartedAt == null && + event.usageMonth == null && + event.usageBytes == null && + event.usageDurationMs == null && + event.usageEffectRetryAt == null && + event.usageEffectLease == null && + event.usageEffectLeaseAt == null && + event.subscriptionEffectState == null && + event.subscriptionEffectLease == null && + event.subscriptionEffectLeaseAt == null && + event.subscriptionEffectRetryAt == null && + event.subscriptionEffectAttemptCount == null && + event.subscriptionEffectDeadLetterAt == null && + event.subscriptionEffectLastError == null + ) +} + +function isPreClaimAuditSnapshot( + event: MailboxDeliveryEventInput | MailboxDeliveryEventRecord, +) { + if ( + event.provider !== mailboxInboundProvider || + event.eventType !== 'rejected' || + typeof event.inboxId !== 'string' || + event.inboxId.length === 0 || + !hasNoInboundAuthorityState(event) + ) { + return false + } + let detail: unknown + try { + detail = JSON.parse(event.detailJson) + } catch { + return false + } + if ( + typeof detail !== 'object' || + detail == null || + !('aggregate' in detail) || + detail.aggregate !== true || + !('day' in detail) || + typeof detail.day !== 'string' || + !auditDayPattern.test(detail.day) || + !('count' in detail) || + typeof detail.count !== 'number' || + !Number.isInteger(detail.count) || + detail.count < 1 || + !('last_reason' in detail) || + typeof detail.last_reason !== 'string' || + !('last_phase' in detail) || + typeof detail.last_phase !== 'string' || + !preClaimAuditPhases.has(detail.last_phase) || + !('last_at' in detail) || + typeof detail.last_at !== 'string' || + !isCanonicalTimestamp(detail.last_at) + ) { + return false + } + return event.id === `email-rejections:${event.inboxId}:${detail.day}` +} + +function readMailboxDeliveryEvent( + sql: SqlStorage, + eventId: string, +): MailboxDeliveryEventRecord | null { + const row = sql + .exec>( + `SELECT * FROM email_delivery_events WHERE id = ? LIMIT 1`, + eventId, + ) + .toArray()[0] + return row ? mapMailboxDeliveryEventRow(row) : null +} + /** * Fence the only D1 → Mailbox USER inbound bridge. Bootstrap accepts a * validated complete snapshot only when the owner-bound row is still missing. @@ -31,9 +142,13 @@ export function shouldSkipMailboxDeliveryEventWrite( ) { const inboundAuthorityEvent = isUserInboundAuthorityEvent(input.event) if (inboundAuthorityEvent && input.intent !== 'user-inbound-bootstrap') { - throw new Error( - 'USER inbound delivery events require the missing-row bootstrap intent.', - ) + if (!isPreClaimAuditSnapshot(input.event)) { + throw new Error( + 'USER inbound delivery events require the missing-row bootstrap intent.', + ) + } + const existing = readMailboxDeliveryEvent(sql, input.event.id) + return existing != null && !isPreClaimAuditSnapshot(existing) } if (!inboundAuthorityEvent && input.intent === 'user-inbound-bootstrap') { throw new Error( @@ -73,13 +188,3 @@ export function shouldSkipMailboxDeliveryEventWrite( .toArray().length > 0 ) } - -export function assertMailboxD1DeliveryEventBatch( - events: Array, -) { - if (events.some(isUserInboundAuthorityEvent)) { - throw new Error( - 'USER inbound delivery events cannot use the D1-authoritative batch mirror.', - ) - } -} diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts index fd0350d209..bdd9548ecc 100644 --- a/packages/worker/src/email/mailbox-reconcile.workers.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -100,18 +100,35 @@ async function seedMessageGraph(input: { } } -async function seedOrphanEvent(input: { +async function seedPreClaimAuditEvent(input: { userId: string - eventId: string + inboxId: string createdAt: string }) { + const day = input.createdAt.slice(0, 10) + const eventId = `email-rejections:${input.inboxId}:${day}` await env.APP_DB.prepare( `INSERT INTO email_delivery_events ( - id, message_id, user_id, event_type, provider, detail_json, created_at - ) VALUES (?, NULL, ?, 'receive_started', 'cloudflare-email-routing', '{}', ?)`, + id, message_id, user_id, inbox_id, event_type, provider, + detail_json, created_at + ) VALUES (?, NULL, ?, ?, 'rejected', 'cloudflare-email-routing', ?, ?)`, ) - .bind(input.eventId, input.userId, input.createdAt) + .bind( + eventId, + input.userId, + input.inboxId, + JSON.stringify({ + aggregate: true, + day, + count: 2, + last_reason: 'Recipient mailbox is over quota.', + last_phase: 'entitlement', + last_at: input.createdAt, + }), + input.createdAt, + ) .run() + return eventId } async function readParityState(userId: string) { @@ -189,9 +206,9 @@ test('reconcileMailboxParity mismatch full reset and soak', async () => { threadId: 'parity-thread-2', createdAt: '2026-07-01T12:01:00.000Z', }) - await seedOrphanEvent({ + const auditEventId = await seedPreClaimAuditEvent({ userId, - eventId: 'parity-orphan-1', + inboxId: 'parity-inbox-1', createdAt: '2026-07-01T12:02:00.000Z', }) await seedMessageGraph({ @@ -206,7 +223,7 @@ test('reconcileMailboxParity mismatch full reset and soak', async () => { const firstNow = new Date('2026-08-01T10:00:00.000Z') vi.setSystemTime(firstNow) - // 2 messages + 2 bound events + 1 orphan + // 2 messages + 2 bound events + 1 bounded pre-claim audit event await expect( reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), ).resolves.toEqual({ @@ -232,7 +249,7 @@ test('reconcileMailboxParity mismatch full reset and soak', async () => { messagesCompletedAt: expect.any(String), eventsCompletedAt: expect.any(String), messageCursorId: 'parity-msg-2', - eventCursorId: 'parity-orphan-1', + eventCursorId: auditEventId, lastError: null, }) From 5e767d6bab7eed3e72b9d0fd73fbc27d7eafc590 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 17:40:18 +0000 Subject: [PATCH 59/62] test(email): cover inbound authority mirror guard Co-authored-by: Kent C. Dodds --- ...ox-inbound-authority-guard.workers.test.ts | 145 ++++++++++++++++++ packages/worker/src/email/service.ts | 5 +- 2 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts diff --git a/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts b/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts new file mode 100644 index 0000000000..304d140974 --- /dev/null +++ b/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts @@ -0,0 +1,145 @@ +import { runInDurableObject } from 'cloudflare:test' +import { expect, test } from 'vitest' +import { insertInput } from './mailbox-inbound-ledger-test-helpers.ts' +import { + assertMailboxThrows, + baseDeliveryEvent, + rpcFor, + stubFor, + uniqueUserId, +} from './mailbox-test-helpers.ts' + +function preClaimAuditEvent(input: { + inboxId: string + day: string + count?: number +}) { + const at = `${input.day}T12:00:00.000Z` + return baseDeliveryEvent({ + id: `email-rejections:${input.inboxId}:${input.day}`, + inboxId: input.inboxId, + eventType: 'rejected', + provider: 'cloudflare-email-routing', + detailJson: JSON.stringify({ + aggregate: true, + day: input.day, + count: input.count ?? 1, + last_reason: 'Recipient mailbox is over quota.', + last_phase: 'entitlement', + last_at: at, + }), + createdAt: at, + updatedAt: at, + }) +} + +test('D1 mirror accepts only non-authoritative bounded inbound rejection audits', async () => { + const ownerId = uniqueUserId('audit-guard') + const mailbox = rpcFor(ownerId) + const audit = preClaimAuditEvent({ + inboxId: 'inbox-audit', + day: '2026-08-02', + }) + + await expect( + mailbox.upsertDeliveryEvent({ ownerId, event: audit }), + ).resolves.toEqual({ + inserted: true, + accepted: true, + updatedLatestStatus: false, + }) + await expect( + mailbox.upsertDeliveryEvents({ + ownerId, + events: [ + { + ...audit, + detailJson: preClaimAuditEvent({ + inboxId: 'inbox-audit', + day: '2026-08-02', + count: 2, + }).detailJson, + }, + ], + }), + ).resolves.toEqual({ + results: [{ eventId: audit.id, inserted: false, accepted: true }], + }) + + await runInDurableObject(stubFor(ownerId), async (instance) => { + await assertMailboxThrows(/missing-row bootstrap intent/, () => + instance.upsertDeliveryEvents({ + ownerId, + events: [ + { + ...audit, + state: 'pending', + fingerprint: 'smuggled-authority-state', + }, + ], + }), + ) + await assertMailboxThrows(/missing-row bootstrap intent/, () => + instance.upsertDeliveryEvents({ + ownerId, + events: [ + baseDeliveryEvent({ + id: 'email-inbound-delivery:lifecycle', + provider: 'cloudflare-email-routing', + eventType: 'receive_started', + state: 'pending', + fingerprint: 'lifecycle-fingerprint', + }), + ], + }), + ) + await assertMailboxThrows(/missing-row bootstrap intent/, () => + instance.upsertDeliveryEvents({ + ownerId, + events: [ + baseDeliveryEvent({ + id: 'email-inbound-dedupe:lifecycle', + provider: 'cloudflare-email-routing-dedupe', + eventType: 'receive_started', + state: 'pending', + fingerprint: 'lifecycle-fingerprint', + }), + ], + }), + ) + }) +}) + +test('audit-shaped D1 snapshots cannot overwrite authoritative Mailbox rows', async () => { + const ownerId = uniqueUserId('audit-collision') + const mailbox = rpcFor(ownerId) + const audit = preClaimAuditEvent({ + inboxId: 'inbox-collision', + day: '2026-08-02', + }) + const authoritative = insertInput(ownerId, { + deliveryId: audit.id, + inboxId: 'inbox-collision', + fingerprint: 'authoritative-fingerprint', + }) + await mailbox.insertChargedPendingInboundDelivery({ + ownerId, + delivery: authoritative, + now: '2026-08-02T11:00:00.000Z', + }) + + await expect( + mailbox.upsertDeliveryEvents({ ownerId, events: [audit] }), + ).resolves.toEqual({ + results: [{ eventId: audit.id, inserted: false, accepted: false }], + }) + await expect( + mailbox.getInboundDelivery({ + ownerId, + deliveryId: audit.id, + }), + ).resolves.toMatchObject({ + state: 'pending', + fingerprint: 'authoritative-fingerprint', + }) +}) diff --git a/packages/worker/src/email/service.ts b/packages/worker/src/email/service.ts index 241fab02a4..1b38941c5b 100644 --- a/packages/worker/src/email/service.ts +++ b/packages/worker/src/email/service.ts @@ -697,8 +697,9 @@ export async function recordBoundedEmailRejectionEvent(input: { const row = await input.db .prepare( `INSERT INTO email_delivery_events ( - id, user_id, inbox_id, event_type, provider, detail_json, created_at - ) VALUES (?, ?, ?, 'rejected', 'cloudflare-email-routing', ?, ?) + id, user_id, inbox_id, event_type, provider, detail_json, + needs_effect_reconcile, created_at + ) VALUES (?, ?, ?, 'rejected', 'cloudflare-email-routing', ?, 0, ?) ON CONFLICT(id) DO UPDATE SET detail_json = json_set( email_delivery_events.detail_json, '$.count', COALESCE(json_extract(email_delivery_events.detail_json, '$.count'), 0) + 1, From c2b7909410d9c8d992a1647b50eeaa0aebba92ff Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 17:54:20 +0000 Subject: [PATCH 60/62] fix(email): repair rejected and dedupe projections Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 46 ++++--- .../inbound-delivery-authority.node.test.ts | 74 ++++++++++ .../src/email/inbound-delivery-authority.ts | 38 +++--- .../inbound-mailbox-mirror.workers.test.ts | 129 ++++++++++++++++++ .../src/email/mailbox-inbound-ledger.ts | 11 +- .../mailbox-inbound-ledger.workers.test.ts | 17 +++ 6 files changed, 273 insertions(+), 42 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 6ea8b43b5d..0c66731bb0 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -725,14 +725,17 @@ claim/release and orphan-cleaned tombstones are also owner-bound CAS. The DO does **not** perform external usage recording or subscription dispatch: Workers claim in Mailbox, perform the D1 usage-rollup or package dispatch, then -complete/fail in Mailbox. After every successful USER CAS, -`inbound-delivery-authority.ts` synchronously upserts one full Mailbox snapshot -into D1 `email_delivery_events` (promoted columns plus `detail_json`, -owner/provider fenced, monotonic and idempotent). Pending is mirrored before -delivery reads; storing is mirrored before D1 thread/message/attachment fence -predicates. Mirror failure fails closed. Terminal/effect snapshots keep D1 -global due-owner discovery current, but D1 is never read back as ongoing -authority. +complete/fail in Mailbox. `inbound-delivery-authority.ts` synchronously upserts +one full Mailbox snapshot into D1 `email_delivery_events` (promoted columns plus +`detail_json`, owner/provider fenced, monotonic and idempotent). Pending is +mirrored before delivery reads; storing is mirrored before D1 +thread/message/attachment fence predicates. Those fence-critical failures fail +closed. A rejected CAS is the deliberate exception: SMTP rejection remains +permanent when its D1 projection fails, and rejected terminal work read-repairs +the projection. Terminal/effect snapshots keep D1 global due-owner discovery +current, but D1 is never read back as ongoing authority. Dedupe pruning projects +only the exact bounded pointer IDs deleted by Mailbox, with D1 owner/provider +fences; it never runs a second independent expiry/limit selection. The only reverse path is the deployment bridge: when a USER point lookup misses in Mailbox and a pre-deploy D1 row exists, one complete D1 snapshot bootstraps @@ -862,15 +865,16 @@ Never throws; returns a bounded summary. **Live callers:** successful D1 update; failures never change the mutation response). - **Inbound terminals** (`inbound.ts`) — USER delivery authority starts in Mailbox: dedupe claim, UserMeter consume, and charged-pending CAS precede - D1/R2 message-graph storage. Mailbox then finalizes `received` or `rejected` - and synchronously projects the full delivery snapshot to D1. A received winner - schedules D1 message-graph repair via `ctx.waitUntil` - (`scheduleInboundReceivedTerminalWork`) without D1 delivery-event write-back. - **Already-received** Email Routing retries (delivery ledger - `state === 'received'` with an existing message row) idempotently repair the - graph and re-run effect reconciliation without a second charge. **Rejected - terminals** (post-claim parse failure or replay of a claimed `rejected` - delivery) read-repair the Mailbox → D1 projection via + D1/R2 message-graph storage. Mailbox then finalizes `received` or `rejected`. + Received snapshots synchronously project to D1; rejected snapshots project + best-effort so a compatibility-write outage cannot undo the permanent SMTP + reject. A received winner schedules D1 message-graph repair via + `ctx.waitUntil` (`scheduleInboundReceivedTerminalWork`) without D1 + delivery-event write-back. **Already-received** Email Routing retries + (delivery ledger `state === 'received'` with an existing message row) + idempotently repair the graph and re-run effect reconciliation without a + second charge. **Rejected terminals** (post-claim parse failure or replay of a + claimed `rejected` delivery) read-repair the Mailbox → D1 projection via `scheduleInboundRejectedTerminalWork`. Effects claim and complete/fail in Mailbox around external work; terminal snapshots synchronously repair D1. Terminal coordinator failures are contained after the authoritative CAS. @@ -1187,9 +1191,11 @@ For USER mail, the owner-bound Mailbox ledger is the lifecycle/effect authority: pending snapshot is synchronously projected to D1. 2. Thread prework, R2 raw-MIME put, and D1 message/attachment storage build the message graph. D1 remains authoritative for that graph. -3. Mailbox CAS finalizes the delivery as `received` or `rejected`; the complete - Mailbox snapshot is synchronously projected to D1. Fence-critical projection - failures fail closed. +3. Mailbox CAS finalizes the delivery as `received` or `rejected`. A received + snapshot is synchronously projected to D1 and fence-critical projection + failures fail closed. Rejection projection is best-effort because Mailbox has + already made the SMTP rejection permanent; rejected terminal work + read-repairs D1. 4. Received terminal work repairs the D1-authoritative message graph into Mailbox without delivery events, runs externally executed effects under Mailbox leases, then read-repairs the Mailbox → D1 projection. diff --git a/packages/worker/src/email/inbound-delivery-authority.node.test.ts b/packages/worker/src/email/inbound-delivery-authority.node.test.ts index 7902c605e3..b43f3d91f8 100644 --- a/packages/worker/src/email/inbound-delivery-authority.node.test.ts +++ b/packages/worker/src/email/inbound-delivery-authority.node.test.ts @@ -755,6 +755,80 @@ test('bootstrap skips malformed and cross-owner legacy D1 rows', async () => { sqlite.close() }) +test('dedupe prune projects only the exact Mailbox IDs with owner/provider fences', async () => { + const sqlite = new DatabaseSync(':memory:') + const db = createD1FromSqlite(sqlite) + await ensureEmailTestSchema(db) + const ownerId = 'owner-prune' + const selectedId = 'email-inbound-dedupe:selected' + const unselectedId = 'email-inbound-dedupe:unselected' + const foreignId = 'email-inbound-dedupe:foreign' + const wrongProviderId = 'email-inbound-dedupe:wrong-provider' + const now = new Date('2026-08-02T12:00:00.000Z') + for (const [id, userId, provider] of [ + [selectedId, ownerId, 'cloudflare-email-routing-dedupe'], + [unselectedId, ownerId, 'cloudflare-email-routing-dedupe'], + [foreignId, 'other-owner', 'cloudflare-email-routing-dedupe'], + [wrongProviderId, ownerId, 'cloudflare-email-routing'], + ] as const) { + await db + .prepare( + `INSERT INTO email_delivery_events ( + id, user_id, event_type, provider, created_at, updated_at + ) VALUES (?, ?, 'receive_started', ?, ?, ?)`, + ) + .bind(id, userId, provider, now.toISOString(), now.toISOString()) + .run() + } + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: db, + USER_METER: namespace({}), + MAILBOX: namespace({ + pruneExpiredInboundDedupePointers: vi.fn(async () => ({ + pruned: 3, + prunedEventIds: [selectedId, foreignId, wrongProviderId], + })), + }), + }, + userId: ownerId, + }) + + expect(await authority.pruneExpiredDedupe(now, 10)).toBe(3) + const remaining = await db + .prepare( + `SELECT id FROM email_delivery_events + ORDER BY id`, + ) + .all<{ id: string }>() + expect(remaining.results?.map((row) => row.id)).toEqual([ + foreignId, + unselectedId, + wrongProviderId, + ]) + sqlite.close() +}) + +test('empty Mailbox dedupe prune does not issue a D1 delete', async () => { + const prepare = vi.fn() + const authority = createUserInboundDeliveryAuthority({ + env: { + APP_DB: { prepare } as unknown as D1Database, + USER_METER: namespace({}), + MAILBOX: namespace({ + pruneExpiredInboundDedupePointers: vi.fn(async () => ({ + pruned: 0, + prunedEventIds: [], + })), + }), + }, + userId: 'owner-empty-prune', + }) + + expect(await authority.pruneExpiredDedupe()).toBe(0) + expect(prepare).not.toHaveBeenCalled() +}) + test('system D1 mutation surface rejects USER deliveries before writing', async () => { const now = new Date('2026-08-02T12:00:00.000Z') const delivery = await buildInboundDelivery({ diff --git a/packages/worker/src/email/inbound-delivery-authority.ts b/packages/worker/src/email/inbound-delivery-authority.ts index 31f64c388e..b8359af16c 100644 --- a/packages/worker/src/email/inbound-delivery-authority.ts +++ b/packages/worker/src/email/inbound-delivery-authority.ts @@ -151,7 +151,7 @@ export type UserInboundDeliveryAuthority = { delivery: InboundDelivery, reason: string, now?: Date, - ) => Promise + ) => Promise receive: (input: UserInboundDeliveryReceiveInput) => Promise deferReconciliation: ( deliveryId: string, @@ -430,12 +430,22 @@ export function createUserInboundDeliveryAuthority( result.status === 'rejected' || result.status === 'already-rejected' ) { - await mirror(result.delivery) - return true + const canonical = toUserInboundDelivery(userId, result.delivery) + try { + return await mirror(result.delivery) + } catch (error) { + console.warn( + 'inbound-email-rejected-projection-failed', + userId, + result.delivery.deliveryId, + error, + ) + return canonical + } } if (result.status === 'already-received') { await mirror(result.delivery) - return false + return null } throw new InboundDeliveryLeaseLostError( 'Inbound rejection lost a state race; delivery should be retried.', @@ -557,23 +567,15 @@ export function createUserInboundDeliveryAuthority( now: now.toISOString(), limit, }) + if (result.prunedEventIds.length === 0) return 0 + const placeholders = result.prunedEventIds.map(() => '?').join(', ') await env.APP_DB.prepare( `DELETE FROM email_delivery_events - WHERE id IN ( - SELECT id FROM email_delivery_events - WHERE user_id = ? - AND provider = ? - AND dedupe_expires_at <= ? - ORDER BY created_at ASC, id ASC - LIMIT ? - )`, + WHERE user_id = ? + AND provider = ? + AND id IN (${placeholders})`, ) - .bind( - userId, - mailboxInboundDedupeProvider, - now.toISOString(), - limit ?? 20, - ) + .bind(userId, mailboxInboundDedupeProvider, ...result.prunedEventIds) .run() return result.pruned }, diff --git a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts index 2de20c4169..1da43d2e4d 100644 --- a/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts +++ b/packages/worker/src/email/inbound-mailbox-mirror.workers.test.ts @@ -1,6 +1,7 @@ import { env } from 'cloudflare:workers' import { expect, test, vi } from 'vitest' import { userMeterRpc } from '#worker/entitlements/user-meter-client.ts' +import { consoleWarn } from '#worker/test-support/console-spies.ts' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { ensureUsageRollupsTestSchema } from '#worker/usage/test-schema.ts' @@ -88,6 +89,55 @@ function createFailingEmailBlobs() { }) } +function createRejectProjectionFailingDb(db: D1Database) { + let failed = false + const failingDb = new Proxy(db, { + get(target, property) { + if (property === 'prepare') { + return (query: string) => { + const statement = target.prepare(query) + if (!query.includes('INSERT INTO email_delivery_events')) { + return statement + } + return new Proxy(statement, { + get(statementTarget, statementProperty) { + if (statementProperty === 'bind') { + return (...values: Array) => { + const bound = statementTarget.bind(...values) + if (failed || values[4] !== 'rejected') return bound + return new Proxy(bound, { + get(boundTarget, boundProperty) { + if (boundProperty === 'run') { + return async () => { + failed = true + throw new Error( + 'simulated rejected projection failure', + ) + } + } + const value = Reflect.get(boundTarget, boundProperty) + return typeof value === 'function' + ? value.bind(boundTarget) + : value + }, + }) + } + } + const value = Reflect.get(statementTarget, statementProperty) + return typeof value === 'function' + ? value.bind(statementTarget) + : value + }, + }) + } + } + const value = Reflect.get(target, property) + return typeof value === 'function' ? value.bind(target) : value + }, + }) + return { db: failingDb, didFail: () => failed } +} + const inboundMailboxMirrorTimeoutMs = 30_000 function createLedgerBackedMailboxStub( @@ -865,3 +915,82 @@ test( }, inboundMailboxMirrorTimeoutMs, ) + +test.each([ + { label: 'without an execution context', withContext: false }, + { label: 'with an execution context', withContext: true }, +])( + 'SMTP rejection survives a rejected D1 projection failure $label', + async ({ withContext }) => { + silenceIncidentalRuntimeWarnings([ + 'inbound-email-rejected-projection-failed', + ]) + await ensureEmailTestSchema(env.APP_DB) + const username = `mbx-rej-fail-${crypto.randomUUID().slice(0, 8)}` + const accountEmail = `mbx-rej-fail-${crypto.randomUUID()}@example.com` + const userId = await createStableUserIdFromEmail(accountEmail) + const address = `${username}@${platformDomain}` + await seedVerifiedAccount({ + db: env.APP_DB, + email: accountEmail, + username, + }) + const mailbox = rpcFor(userId) + await mailbox.getMessage({ messageId: 'warmup-nonexistent' }) + const raw = [ + 'From: Sender ', + `To: ${address}`, + 'Subject: Permanent reject despite projection failure', + `Message-ID: `, + '', + 'Body', + ].join('\r\n') + const projection = createRejectProjectionFailingDb(env.APP_DB) + const inboundEnv = { + ...createInboundEnv(), + APP_DB: projection.db, + } + const captured = withContext ? createCapturedWaitUntilContext() : null + const parseSpy = vi + .spyOn(parser, 'parseForwardableEmailRawMime') + .mockRejectedValueOnce(new Error('permanent parse rejection')) + try { + const message = createForwardableEmailMessage({ + from: 'sender@example.net', + to: address, + raw, + }) + await handleInboundEmail(message, inboundEnv, captured?.ctx) + + expect(projection.didFail()).toBe(true) + expect(message.rejectedReason).toBe('permanent parse rejection') + expect(consoleWarn).toHaveBeenCalledWith( + 'inbound-email-rejected-projection-failed', + userId, + expect.any(String), + expect.any(Error), + ) + if (captured) await drainWaitUntil(captured.waitUntilPromises) + + const rejected = ( + await mailbox.listDeliveryEvents({ messageId: null, limit: 20 }) + ).find((event) => event.eventType === 'rejected') + expect(rejected).toMatchObject({ + eventType: 'rejected', + provider: 'cloudflare-email-routing', + }) + if (!rejected) throw new Error('Expected authoritative Mailbox rejection') + expect( + await env.APP_DB.prepare( + `SELECT event_type FROM email_delivery_events + WHERE id = ? AND user_id = ?`, + ) + .bind(rejected.id, userId) + .first<{ event_type: string }>(), + ).toEqual({ event_type: 'rejected' }) + } finally { + parseSpy.mockRestore() + } + }, + inboundMailboxMirrorTimeoutMs, +) diff --git a/packages/worker/src/email/mailbox-inbound-ledger.ts b/packages/worker/src/email/mailbox-inbound-ledger.ts index d0f971865c..44393939b2 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.ts @@ -117,7 +117,10 @@ export type MailboxMarkInboundDeliveryOrphanCleanedResult = | { status: 'orphan-cleaned'; delivery: MailboxInboundDeliverySnapshot } | { status: 'lease-lost' } -export type MailboxPruneExpiredInboundDedupeResult = { pruned: number } +export type MailboxPruneExpiredInboundDedupeResult = { + pruned: number + prunedEventIds: Array +} export type MailboxListDueStaleInboundDeliveriesResult = { deliveries: Array @@ -787,7 +790,7 @@ export function pruneMailboxExpiredInboundDedupePointers( ) .toArray() .map((row) => String(row.id)) - let pruned = 0 + const prunedEventIds: Array = [] for (const id of ids) { const cursor = sql.exec( `DELETE FROM email_delivery_events @@ -796,9 +799,9 @@ export function pruneMailboxExpiredInboundDedupePointers( mailboxInboundDedupeProvider, now, ) - pruned += cursor.rowsWritten + if (cursor.rowsWritten > 0) prunedEventIds.push(id) } - return { pruned } + return { pruned: prunedEventIds.length, prunedEventIds } } export function deferMailboxInboundDeliveryReconciliation( diff --git a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts index 7b30dc43a0..83e623d6e7 100644 --- a/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-ledger.workers.test.ts @@ -5,6 +5,7 @@ import { emailRawMimeKey } from './blob-keys.ts' import { Mailbox } from './mailbox-do.ts' import { insertInput } from './mailbox-inbound-ledger-test-helpers.ts' import { + mailboxInboundDedupePointerId, mailboxInboundDedupeProvider, mailboxInboundProvider, mailboxInboundReconciliationRetryMs, @@ -331,6 +332,11 @@ test('Mailbox inbound ledger CAS covers USER authority transition matrix', async messageId: 'email-inbound-message:owner-b', threadId: 'email-inbound-thread:owner-b', }) + await mailboxB.claimInboundDeliveryWindow({ + ownerId: ownerB, + delivery: foreign, + now, + }) await mailboxB.insertChargedPendingInboundDelivery({ ownerId: ownerB, delivery: foreign, @@ -369,6 +375,17 @@ test('Mailbox inbound ledger CAS covers USER authority transition matrix', async limit: 50, }) expect(pruned.pruned).toBeGreaterThan(0) + expect(pruned.prunedEventIds).toHaveLength(pruned.pruned) + expect(pruned.prunedEventIds).not.toContain( + mailboxInboundDedupePointerId(foreign.fingerprint), + ) + expect( + await mailboxB.getInboundDeliveryWindow({ + ownerId: ownerB, + fingerprint: foreign.fingerprint, + now, + }), + ).toMatchObject({ deliveryId: foreign.deliveryId }) // Defer reconcile. const deferred = await mailboxA.deferInboundDeliveryReconciliation({ From a5f39529120190b99a775320a28010bc8b0f6c37 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 18:41:31 +0000 Subject: [PATCH 61/62] fix(email): bootstrap legacy inbound parity rows Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 154 +++++++--- docs/contributing/disaster-recovery.md | 13 + .../src/email/inbound-delivery-authority.ts | 4 + .../src/email/inbound-delivery-projection.ts | 38 +-- .../email/mailbox-delivery-event-bootstrap.ts | 62 ++++ .../email/mailbox-delivery-event-upsert.ts | 33 ++ packages/worker/src/email/mailbox-do.ts | 55 ++-- .../src/email/mailbox-do.workers.test.ts | 87 ------ ...ox-inbound-authority-guard.workers.test.ts | 150 +++++++++- .../src/email/mailbox-inbound-bootstrap.ts | 151 +++++++--- .../src/email/mailbox-mirror.node.test.ts | 81 +++++ packages/worker/src/email/mailbox-mirror.ts | 183 ++++++++--- .../worker/src/email/mailbox-parity-phases.ts | 6 +- .../email/mailbox-reconcile.workers.test.ts | 283 ++++++++++++++++++ packages/worker/src/email/mailbox-types.ts | 26 +- 15 files changed, 1054 insertions(+), 272 deletions(-) create mode 100644 packages/worker/src/email/mailbox-delivery-event-bootstrap.ts create mode 100644 packages/worker/src/email/mailbox-delivery-event-upsert.ts diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 0c66731bb0..5675f9f19d 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -740,21 +740,93 @@ fences; it never runs a second independent expiry/limit selection. The only reverse path is the deployment bridge: when a USER point lookup misses in Mailbox and a pre-deploy D1 row exists, one complete D1 snapshot bootstraps the owner-bound DO row, after which transitions continue through Mailbox CAS. -`system:email` stays on the existing D1 implementation and cannot bootstrap a -Mailbox. Migration `0129-email-inbound-mailbox-authority-mirror.sql` is -additive: it promotes compatibility/fence fields and adds the cross-store usage -effect idempotency ledger; it drops no tables. Destructive follow-up work -remains gated on the verified backup whose SHA-256 starts with `7787f8c9`; this -change is explicitly non-destructive. - -**Mirror write contract:** `mirrorMessage`, `upsertDeliveryEvent`, and -`upsertDeliveryEvents` take complete snapshots — every persisted field is -explicit (nullable fields use explicit `null`). They are not patch APIs. All -require `ownerId` and apply equal-or-newer `updatedAt` snapshots (stale -snapshots are ignored, not applied). `upsertDeliveryEvents` accepts at most -`mailboxUpsertDeliveryEventsMax` (100) events in one owner-bound batch RPC and -inserts them in caller order (chronological when loaded from D1). The only -omission exception is the `mirrorMessage` `attachments` bundle: omitting it +Scheduled parity partitions those validated legacy lifecycle/dedupe snapshots to +the missing-only `bootstrapDeliveryEvents` RPC; pre-claim rejection audits and +non-inbound events continue through normal `upsertDeliveryEvents`. The bootstrap +RPC accepts at most 100 snapshots, validates owner/provider/detail coherence, +and reports inserted/existing/skipped counts. It never updates an existing +Mailbox row. Normal delivery-event upserts continue rejecting USER inbound +authority snapshots. `system:email` stays on the existing D1 implementation and +cannot bootstrap a Mailbox. Migration +`0129-email-inbound-mailbox-authority-mirror.sql` is additive: it promotes +compatibility/fence fields and adds the cross-store usage effect idempotency +ledger; it drops no tables. Destructive follow-up work remains gated on the +verified backup whose SHA-256 starts with `7787f8c9`; this change is explicitly +non-destructive. + +**Accepted rollback → roll-forward caveat and manual repair:** a rollback to the +previous Worker can advance USER inbound state in legacy `detail_json` with +`json_set` without advancing `email_delivery_events.updated_at`. A later +roll-forward does not automatically detect that D1 is newer: the bootstrap is +missing-only, an existing Mailbox authority row wins, and the Mailbox → D1 +projection fence treats existing D1 `updated_at >=` the snapshot timestamp as +already current. Rollback-era D1 progress can therefore require operator repair +before redeploy. + +Use this exact owner-by-owner procedure; do **not** run ordinary Mailbox purge +casually: + +1. Gate the repair on the sealed, verified production backup whose D1 SQL + SHA-256 starts with `7787f8c9`. Verify the signed manifest and stored object, + not a copied checksum string. Stop if the prefix or restore drill evidence + does not match. +2. Put the app behind the approved maintenance controls, disable the affected + Cloudflare Email Routing ingress, and pause scheduled/queue consumers that + can write email. Keep the rollback Worker quiesced for the entire inspection, + purge, rebuild, and verification window. +3. For each affected `stable_user_id`, capture D1 `email_threads`, + `email_messages`, `email_attachments`, and `email_delivery_events` counts. + Inspect every inbound lifecycle/dedupe row's `detail_json`, promoted state, + effect/finalization fields, `created_at`, and `updated_at`; compare with + `Mailbox.exportMailbox`/`countMailbox`. Decide that D1 contains the desired + rollback-era progress from operator evidence. The code does not make this + decision. +4. Only after that owner passes inspection, invoke the existing owner-derived + `Mailbox.purge()` RPC through a reviewed production operator script/Worker + using `MAILBOX.idFromName(stable_user_id)`. This is the safe metadata-only + purge: it clears that owner's Mailbox SQLite/alarm state and does not delete + D1 or R2. Never substitute the normal retention/delete surfaces. +5. Reset only that owner's parity state in D1, preserving all `email_*` rows: + + ```sql + UPDATE users + SET mailbox_parity_checked_at = NULL, + mailbox_parity_matching_since = NULL, + mailbox_parity_mismatch_count = 0, + mailbox_parity_last_error = NULL, + mailbox_parity_content_watermark_at = NULL, + mailbox_parity_content_replay_upper_at = NULL, + mailbox_parity_content_replay_cursor_updated_at = NULL, + mailbox_parity_content_replay_cursor_id = NULL, + mailbox_parity_message_backfill_cursor_created_at = NULL, + mailbox_parity_message_backfill_cursor_id = NULL, + mailbox_parity_message_backfill_completed_at = NULL, + mailbox_parity_event_backfill_cursor_created_at = NULL, + mailbox_parity_event_backfill_cursor_id = NULL, + mailbox_parity_event_backfill_completed_at = NULL + WHERE stable_user_id = ? AND deleting_at IS NULL; + ``` + +6. Run `admin_mailbox_maintenance({ action: "reconcile", batch_size: 100 })` + until that owner completes a full D1 → Mailbox rebuild and exact count + compare. Re-export the Mailbox and verify per-row lifecycle state, dedupe + pointers, effect state/leases/retry/dead-letter fields, finalization tokens, + usage fields, and message/attachment counts against the inspected D1 source. + Require zero parity error/mismatch before continuing. +7. Redeploy the roll-forward Worker while writes remain quiesced. Re-run + owner/fleet status and a focused inbound canary, then resume queues, + schedules, Email Routing, and normal ingress in that order. + +**Mirror write contract:** `mirrorMessage`, `upsertDeliveryEvent`, +`upsertDeliveryEvents`, and `bootstrapDeliveryEvents` take complete snapshots — +every persisted field is explicit (nullable fields use explicit `null`). They +are not patch APIs. All require `ownerId`. Normal upserts apply equal-or-newer +`updatedAt` snapshots (stale snapshots are ignored) and reject USER inbound +lifecycle/dedupe authority rows. `bootstrapDeliveryEvents` is their explicit +missing-only exception: it validates legacy USER inbound snapshots and never +updates an existing ID. Both batch RPCs accept at most +`mailboxUpsertDeliveryEventsMax` (100) events and process in caller order. The +only omission exception is the `mirrorMessage` `attachments` bundle: omitting it preserves existing attachment rows; an explicit `attachments: []` clears them. Accepted mirrors validate inbound/outbound `rawMimeKey` and external attachment `storageKey` values against the canonical builders for that `ownerId`. @@ -813,16 +885,18 @@ batch bound without slowing live dual-write. Each returns a structured `MailboxMirrorResult`: `{ status: 'mirrored' }`, `{ status: 'stale' }`, `{ status: 'missing' }`, `{ status: 'timeout' }`, `{ status: 'skipped', reason }` (`system-email` | `mailbox-unconfigured` | -`missing-owner`), or `{ status: 'error', error }`. Single-RPC helpers record one -`mailbox_mirror:` outcome automatically when a user id is known; -`mirrorMailboxDeliveryEventSnapshots` records one -`mailbox_mirror:upsert_delivery_event_batch` outcome per batch (timeout/error -apply uniformly to per-event summary entries). `system:email` is excluded. -Failures log with stable tags (for example `mailbox-mirror-message-failed`) and -never propagate into D1 commit paths. Helpers cover full snapshots -(`mirrorMailboxMessageSnapshot`, `mirrorMailboxDeliveryEventSnapshot`, -`mirrorMailboxDeliveryEventSnapshots` — prefer loading delivery events with -`getMailboxDeliveryEventMirrorInput` or +`missing-owner` | `user-inbound-authority`), or `{ status: 'error', error }`. +Single-RPC helpers record one `mailbox_mirror:` outcome automatically +when a user id is known; `mirrorMailboxDeliveryEventSnapshots` partitions a +mixed page into normal and bootstrap subsets, bounds each non-empty RPC, and +records one aggregate `mailbox_mirror:upsert_delivery_event_batch` outcome for +the original page. Inserted/existing/skipped bootstrap results map to +mirrored/stale/skipped rather than treating idempotency as an error. +`system:email` is excluded. Failures log with stable tags (for example +`mailbox-mirror-message-failed`) and never propagate into D1 commit paths. +Helpers cover full snapshots (`mirrorMailboxMessageSnapshot`, +`mirrorMailboxDeliveryEventSnapshot`, `mirrorMailboxDeliveryEventSnapshots` — +prefer loading delivery events with `getMailboxDeliveryEventMirrorInput` or `listMailboxDeliveryEventMirrorInputsForMessage`) and partial mutations (`mirrorMailboxTouchThread`, `mirrorMailboxUpdateMessageDelivery`, `mirrorMailboxSetMessageClassification`, `mirrorMailboxDeleteMessageMetadata`, @@ -835,17 +909,16 @@ synchronous reverse compatibility mirror described above. **Live graph orchestrator** (`mailbox-live-mirror.ts`): loads a cohesive D1 message graph (optional caller thread, message, attachments, then delivery events) and mirrors it best-effort. `mirrorMailboxMessageGraphFromD1` settles -the message snapshot RPC first, then repairs delivery events with one -owner-bound `upsertDeliveryEvents` batch RPC (not concurrent per-event RPCs to -the same DO). Event load queries newest `max+1` rows from D1, restores -chronological order, and when truncated keeps the newest -`mailboxLiveMirrorMaxEvents` (`mailboxUpsertDeliveryEventsMax`, 100) — dropping -oldest overflow — with a stable warning (`mailbox-live-mirror-events-truncated`; -`userId`, `messageId`, `loaded`, `max`). The batch RPC shares one 1s timeout and -one `mailbox_mirror:upsert_delivery_event_batch` telemetry outcome -(timeout/error apply uniformly to per-event summary entries). Each graph attempt -emits at most two Analytics Engine writes (1 message outcome + 1 batch outcome). -Never throws; returns a bounded summary. **Live callers:** +the message snapshot RPC first, then repairs delivery events with sequential +owner-bound normal/bootstrap batch RPCs (never concurrent per-event RPCs to the +same DO). Event load queries newest `max+1` rows from D1, restores chronological +order, and when truncated keeps the newest `mailboxLiveMirrorMaxEvents` +(`mailboxUpsertDeliveryEventsMax`, 100) — dropping oldest overflow — with a +stable warning (`mailbox-live-mirror-events-truncated`; `userId`, `messageId`, +`loaded`, `max`). Each non-empty subset uses the 1s timeout; the original page +emits one `mailbox_mirror:upsert_delivery_event_batch` telemetry outcome. Each +graph attempt emits at most two Analytics Engine writes (1 message outcome + 1 +batch outcome). Never throws; returns a bounded summary. **Live callers:** - **Outbound terminals** (`outbound.ts`) — after D1 reaches a terminal outbound state (`sent`, attachment-store `failed`, or send `failed`), mirrors the full @@ -928,9 +1001,12 @@ Per user, the lane: **every** owner `email_delivery_events` row by `(created_at, id)` into ready `MailboxDeliveryEventInput` snapshots (`listMailboxDeliveryEventMirrorInputsForOwnerKeyset`; not only - `message_id`-null orphans) and mirrors each page with **one** - `upsertDeliveryEvents` batch via `mirrorMailboxDeliveryEventSnapshots` - (`mailboxParityEventPageSize` ≤ DO max, timeout + `message_id`-null orphans). Each page partitions legacy USER inbound + lifecycle/dedupe snapshots to missing-only `bootstrapDeliveryEvents`, and + sends pre-claim rejection audits plus non-inbound rows to normal + `upsertDeliveryEvents`; a legacy authority row cannot roll back the normal + audit batch. Both subsets remain bounded by the original page + (`mailboxParityEventPageSize` ≤ DO max), with timeout `mailboxParityEventMirrorTimeoutMs` ≈ 5s). Production evidence: per-event 1s RPCs repeatedly timed out on a lagging owner and prevented soak under the 10s lane budget; page batches restore convergence while live dual-write keeps the diff --git a/docs/contributing/disaster-recovery.md b/docs/contributing/disaster-recovery.md index 284f4a3b20..c5dd911ebb 100644 --- a/docs/contributing/disaster-recovery.md +++ b/docs/contributing/disaster-recovery.md @@ -359,6 +359,19 @@ Disable ingress / put the app in maintenance before execute. After restore: reindex Vectorize, re-arm jobs/alarms from D1, recreate queues from Wrangler config, and expect users to reauthorize OAuth and remote connectors. +### Mailbox authority rollback repair + +Rollback from the USER inbound Mailbox-authority Worker to its predecessor has +an accepted roll-forward caveat: legacy `json_set` lifecycle writes do not bump +`updated_at`, and the roll-forward does not auto-detect newer D1 state when a +Mailbox row already exists. Before a roll-forward after such a rollback, use the +backup-gated, owner-by-owner metadata purge and full D1 parity rebuild procedure +in +[Data storage → Mailbox](./architecture/data-storage.md#durable-objects-mailbox). +That procedure is gated on the verified backup SHA-256 prefix `7787f8c9`, keeps +the rollback Worker quiesced, and requires effect/finalization verification. Do +not use normal purge as an exploratory or routine repair. + ## Schedules and freshness | When (UTC) | Who | What | diff --git a/packages/worker/src/email/inbound-delivery-authority.ts b/packages/worker/src/email/inbound-delivery-authority.ts index b8359af16c..2eaddd163b 100644 --- a/packages/worker/src/email/inbound-delivery-authority.ts +++ b/packages/worker/src/email/inbound-delivery-authority.ts @@ -300,6 +300,10 @@ export function createUserInboundDeliveryAuthority( } const updatedAt = chargeInput.now.toISOString() const meter = userMeterRpc({ env, userId }) + // Accepted non-atomic cross-DO window: if Email Routing exhausts retries + // after UserMeter consume but before Mailbox insert, one daily receive unit + // may burn until reset. Replays self-heal while retries continue and cannot + // duplicate a message because both operations use the dedupe winner id. let meterResult = await meter.consumeInboundDelivery({ deliveryId: chargedDelivery.deliveryId, resource: 'email_receives_per_day', diff --git a/packages/worker/src/email/inbound-delivery-projection.ts b/packages/worker/src/email/inbound-delivery-projection.ts index 629147fc01..5c1262134c 100644 --- a/packages/worker/src/email/inbound-delivery-projection.ts +++ b/packages/worker/src/email/inbound-delivery-projection.ts @@ -327,16 +327,17 @@ export async function bootstrapUserInboundDeliveryFromD1(input: { return null } const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) - await mailbox.upsertDeliveryEvent({ + await mailbox.bootstrapDeliveryEvents({ ownerId: input.userId, - intent: 'user-inbound-bootstrap', - event: toMailboxBootstrapEvent({ - delivery, - eventId: input.deliveryId, - provider: mailboxInboundProvider, - createdAt: timestamps.created_at, - updatedAt: timestamps.updated_at, - }), + events: [ + toMailboxBootstrapEvent({ + delivery, + eventId: input.deliveryId, + provider: mailboxInboundProvider, + createdAt: timestamps.created_at, + updatedAt: timestamps.updated_at, + }), + ], }) return await mailbox.getInboundDelivery({ ownerId: input.userId, @@ -373,16 +374,17 @@ export async function bootstrapUserInboundDeliveryWindowFromD1(input: { return null } const mailbox = mailboxRpc({ env: input.env, userId: input.userId }) - await mailbox.upsertDeliveryEvent({ + await mailbox.bootstrapDeliveryEvents({ ownerId: input.userId, - intent: 'user-inbound-bootstrap', - event: toMailboxBootstrapEvent({ - delivery, - eventId, - provider: mailboxInboundDedupeProvider, - createdAt: timestamps.created_at, - updatedAt: timestamps.updated_at, - }), + events: [ + toMailboxBootstrapEvent({ + delivery, + eventId, + provider: mailboxInboundDedupeProvider, + createdAt: timestamps.created_at, + updatedAt: timestamps.updated_at, + }), + ], }) return await mailbox.getInboundDeliveryWindow({ ownerId: input.userId, diff --git a/packages/worker/src/email/mailbox-delivery-event-bootstrap.ts b/packages/worker/src/email/mailbox-delivery-event-bootstrap.ts new file mode 100644 index 0000000000..40572dac5c --- /dev/null +++ b/packages/worker/src/email/mailbox-delivery-event-bootstrap.ts @@ -0,0 +1,62 @@ +import { writeMailboxDeliveryEventRow } from './mailbox-delivery-events.ts' +import { assertUserInboundLegacyBootstrapSnapshot } from './mailbox-inbound-bootstrap.ts' +import { + assertMailboxNonEmptyString, + mailboxUpsertDeliveryEventsMax, + type MailboxBootstrapDeliveryEventsResult, + type MailboxDeliveryEventInput, +} from './mailbox-types.ts' + +/** + * Missing-only legacy USER inbound bootstrap inside the caller's transaction. + * Validation failure throws so transactionSync rolls back every prior insert. + */ +export function bootstrapMailboxDeliveryEvents( + sql: SqlStorage, + input: { + ownerId: string + events: Array + }, +): MailboxBootstrapDeliveryEventsResult { + if (!Array.isArray(input.events) || input.events.length === 0) { + throw new Error('Mailbox bootstrapDeliveryEvents events must be non-empty.') + } + if (input.events.length > mailboxUpsertDeliveryEventsMax) { + throw new Error( + `Mailbox bootstrapDeliveryEvents events exceed max of ${mailboxUpsertDeliveryEventsMax}.`, + ) + } + const result: MailboxBootstrapDeliveryEventsResult = { + inserted: 0, + existing: 0, + skipped: 0, + results: [], + } + for (const event of input.events) { + assertUserInboundLegacyBootstrapSnapshot({ + ownerId: input.ownerId, + event, + }) + const eventId = assertMailboxNonEmptyString(event.id, 'event.id') + const existing = sql + .exec( + `SELECT id FROM email_delivery_events WHERE id = ? LIMIT 1`, + eventId, + ) + .toArray() + if (existing.length > 0) { + result.existing += 1 + result.results.push({ eventId, status: 'existing' }) + continue + } + const write = writeMailboxDeliveryEventRow(sql, event) + if (write.inserted) { + result.inserted += 1 + result.results.push({ eventId, status: 'inserted' }) + } else { + result.skipped += 1 + result.results.push({ eventId, status: 'skipped' }) + } + } + return result +} diff --git a/packages/worker/src/email/mailbox-delivery-event-upsert.ts b/packages/worker/src/email/mailbox-delivery-event-upsert.ts new file mode 100644 index 0000000000..41ed5f9431 --- /dev/null +++ b/packages/worker/src/email/mailbox-delivery-event-upsert.ts @@ -0,0 +1,33 @@ +import { writeMailboxDeliveryEventRow } from './mailbox-delivery-events.ts' +import { shouldSkipMailboxDeliveryEventWrite } from './mailbox-inbound-bootstrap.ts' +import { + assertMailboxNonEmptyString, + mailboxUpsertDeliveryEventsMax, + type MailboxDeliveryEventInput, + type MailboxUpsertDeliveryEventsResult, +} from './mailbox-types.ts' + +/** Normal bounded delivery-event upsert inside the caller's transaction. */ +export function upsertMailboxDeliveryEvents( + sql: SqlStorage, + events: Array, +): MailboxUpsertDeliveryEventsResult { + if (!Array.isArray(events) || events.length === 0) { + throw new Error('Mailbox upsertDeliveryEvents events must be non-empty.') + } + if (events.length > mailboxUpsertDeliveryEventsMax) { + throw new Error( + `Mailbox upsertDeliveryEvents events exceed max of ${mailboxUpsertDeliveryEventsMax}.`, + ) + } + const results: MailboxUpsertDeliveryEventsResult['results'] = [] + for (const event of events) { + const eventId = assertMailboxNonEmptyString(event.id, 'event.id') + if (shouldSkipMailboxDeliveryEventWrite(sql, { event })) { + results.push({ eventId, inserted: false, accepted: false }) + continue + } + results.push({ eventId, ...writeMailboxDeliveryEventRow(sql, event) }) + } + return { results } +} diff --git a/packages/worker/src/email/mailbox-do.ts b/packages/worker/src/email/mailbox-do.ts index 9ece7acf35..3d3b4fba1f 100644 --- a/packages/worker/src/email/mailbox-do.ts +++ b/packages/worker/src/email/mailbox-do.ts @@ -26,6 +26,8 @@ import { markMailboxInboundDeliveryOrphanCleaned, releaseMailboxInboundDeliveryCleanup, } from './mailbox-inbound-cleanup-ledger.ts' +import { bootstrapMailboxDeliveryEvents } from './mailbox-delivery-event-bootstrap.ts' +import { upsertMailboxDeliveryEvents } from './mailbox-delivery-event-upsert.ts' import { shouldSkipMailboxDeliveryEventWrite } from './mailbox-inbound-bootstrap.ts' import { claimMailboxInboundDeliveryStorage, @@ -52,10 +54,10 @@ import { } from './mailbox-inbound-effect-ledger.ts' import { assertMailboxNonEmptyString, - mailboxUpsertDeliveryEventsMax, type MailboxAttachmentInput, type MailboxAttachmentRecord, type MailboxBlobReferencePage, + type MailboxBootstrapDeliveryEventsResult, type MailboxCountMessagesInput, type MailboxCountResult, type MailboxDeleteDeliveryEventInput, @@ -78,7 +80,6 @@ import { type MailboxThreadRecord, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, - type MailboxUpsertDeliveryEventBatchItemResult, type MailboxUpsertDeliveryEventsResult, } from './mailbox-types.ts' @@ -244,7 +245,6 @@ class MailboxBase extends DurableObject implements MailboxRpc { async upsertDeliveryEvent(input: { ownerId: string event: MailboxDeliveryEventInput - intent?: 'user-inbound-bootstrap' latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus @@ -262,10 +262,7 @@ class MailboxBase extends DurableObject implements MailboxRpc { this.store.assertOwner(input.ownerId) if ( shouldSkipMailboxDeliveryEventWrite(this.ctx.storage.sql, { - ownerId: input.ownerId, event: input.event, - intent: input.intent, - hasLatestDeliveryStatus: input.latestDeliveryStatus != null, }) ) { return @@ -295,36 +292,32 @@ class MailboxBase extends DurableObject implements MailboxRpc { ownerId: string events: Array }): Promise { - if (!Array.isArray(input.events) || input.events.length === 0) { - throw new Error('Mailbox upsertDeliveryEvents events must be non-empty.') - } - if (input.events.length > mailboxUpsertDeliveryEventsMax) { - throw new Error( - `Mailbox upsertDeliveryEvents events exceed max of ${mailboxUpsertDeliveryEventsMax}.`, - ) - } - const results: Array = [] + let result: MailboxUpsertDeliveryEventsResult | undefined this.ctx.storage.transactionSync(() => { this.store.assertOwner(input.ownerId) - for (const event of input.events) { - const eventId = assertMailboxNonEmptyString(event.id, 'event.id') - if ( - shouldSkipMailboxDeliveryEventWrite(this.ctx.storage.sql, { - ownerId: input.ownerId, - event, - hasLatestDeliveryStatus: false, - }) - ) { - results.push({ eventId, inserted: false, accepted: false }) - continue - } - const write = this.store.writeDeliveryEventRow(event) - results.push({ eventId, ...write }) - } + result = upsertMailboxDeliveryEvents(this.ctx.storage.sql, input.events) }) + if (!result) throw new Error('Mailbox upsert transaction did not run.') this.markRetentionDirty() await this.ensureRetentionAlarm() - return { results } + return result + } + + async bootstrapDeliveryEvents(input: { + ownerId: string + events: Array + }): Promise { + let result: MailboxBootstrapDeliveryEventsResult | undefined + this.ctx.storage.transactionSync(() => { + this.store.assertOwner(input.ownerId) + result = bootstrapMailboxDeliveryEvents(this.ctx.storage.sql, input) + }) + if (!result) throw new Error('Mailbox bootstrap transaction did not run.') + if (result.inserted > 0) { + this.markRetentionDirty() + await this.ensureRetentionAlarm() + } + return result } /** diff --git a/packages/worker/src/email/mailbox-do.workers.test.ts b/packages/worker/src/email/mailbox-do.workers.test.ts index 3d83108a9f..e7e752ef87 100644 --- a/packages/worker/src/email/mailbox-do.workers.test.ts +++ b/packages/worker/src/email/mailbox-do.workers.test.ts @@ -496,93 +496,6 @@ test('Mailbox delivery status, promoted inbound fields, export paging, and curso updatedLatestStatus: true, }) - const inboundDetail = { - fingerprint: 'fp-abc', - deliveryId: 'inbound-delivery-1', - messageId: 'inbound-msg-1', - threadId: 'inbound-thread-1', - rawMimeKey: emailRawMimeKey(userId, 'inbound-msg-1'), - userId, - inboxId: 'inbox-1', - recipient: 'owner@example.com', - envelopeFrom: 'sender@example.com', - provider: 'cloudflare-email-routing', - quotaDay: '2026-07-02', - dedupeExpiresAt: '2026-07-04T11:00:00.000Z', - state: 'received' as const, - } - const inbound = await mailbox.upsertDeliveryEvent({ - ownerId: userId, - intent: 'user-inbound-bootstrap', - event: baseDeliveryEvent({ - id: 'inbound-delivery-1', - messageId: 'inbound-msg-1', - inboxId: 'inbox-1', - eventType: 'received', - provider: 'cloudflare-email-routing', - createdAt: '2026-07-02T11:00:00.000Z', - updatedAt: '2026-07-02T11:00:00.000Z', - needsEffectReconcile: true, - state: 'received', - fingerprint: 'fp-abc', - expectedAttachmentCount: 2, - finalizationToken: 'lease-token-1', - usageStartedAt: '2026-07-02T10:59:00.000Z', - usageMonth: '2026-07', - usageBytes: 2048, - usageDurationMs: 120, - subscriptionEffectState: 'pending', - subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', - detailJson: JSON.stringify(inboundDetail), - }), - }) - expect(inbound.inserted).toBe(true) - const inboundRow = ( - await mailbox.listDeliveryEvents({ - messageId: 'inbound-msg-1', - limit: 1, - }) - )[0] - expect(inboundRow).toMatchObject({ - id: 'inbound-delivery-1', - needsEffectReconcile: true, - state: 'received', - fingerprint: 'fp-abc', - expectedAttachmentCount: 2, - finalizationToken: 'lease-token-1', - usageStartedAt: '2026-07-02T10:59:00.000Z', - usageMonth: '2026-07', - usageBytes: 2048, - subscriptionEffectState: 'pending', - subscriptionEffectRetryAt: '2026-07-02T12:00:00.000Z', - updatedAt: '2026-07-02T11:00:00.000Z', - }) - expect(JSON.parse(inboundRow!.detailJson)).toMatchObject({ - recipient: 'owner@example.com', - }) - const repeatedBootstrap = await mailbox.upsertDeliveryEvent({ - ownerId: userId, - intent: 'user-inbound-bootstrap', - event: baseDeliveryEvent({ - id: 'inbound-delivery-1', - messageId: 'inbound-msg-1', - inboxId: 'inbox-1', - eventType: 'rejected', - provider: 'cloudflare-email-routing', - createdAt: '2026-07-02T11:00:00.000Z', - updatedAt: '2026-07-02T12:00:00.000Z', - needsEffectReconcile: false, - state: 'rejected', - fingerprint: 'fp-abc', - detailJson: JSON.stringify({ ...inboundDetail, state: 'rejected' }), - }), - }) - expect(repeatedBootstrap).toEqual({ - inserted: false, - accepted: false, - updatedLatestStatus: false, - }) - // Explicit needsEffectReconcile: false is stored as false. await mailbox.upsertDeliveryEvent({ ownerId: userId, diff --git a/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts b/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts index 304d140974..ec4ab8686f 100644 --- a/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts @@ -1,6 +1,8 @@ import { runInDurableObject } from 'cloudflare:test' import { expect, test } from 'vitest' +import { emailRawMimeKey } from './blob-keys.ts' import { insertInput } from './mailbox-inbound-ledger-test-helpers.ts' +import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' import { assertMailboxThrows, baseDeliveryEvent, @@ -33,6 +35,39 @@ function preClaimAuditEvent(input: { }) } +function legacyPendingEvent(ownerId: string, deliveryId: string) { + const messageId = `email-inbound-message:${deliveryId}` + const fingerprint = `fingerprint-${deliveryId}` + const detail = { + fingerprint, + deliveryId, + messageId, + threadId: `email-inbound-thread:${deliveryId}`, + rawMimeKey: emailRawMimeKey(ownerId, messageId), + userId: ownerId, + inboxId: 'inbox-legacy', + recipient: 'owner@example.com', + envelopeFrom: 'sender@example.com', + provider: 'cloudflare-email-routing', + quotaDay: '2026-08-02', + dedupeExpiresAt: '2026-08-04T12:00:00.000Z', + state: 'pending' as const, + } + return baseDeliveryEvent({ + id: deliveryId, + inboxId: detail.inboxId, + eventType: 'receive_started', + provider: 'cloudflare-email-routing', + providerEventId: deliveryId, + detailJson: JSON.stringify(detail), + state: 'pending', + fingerprint, + dedupeExpiresAt: detail.dedupeExpiresAt, + createdAt: '2026-08-02T12:00:00.000Z', + updatedAt: '2026-08-02T12:00:00.000Z', + }) +} + test('D1 mirror accepts only non-authoritative bounded inbound rejection audits', async () => { const ownerId = uniqueUserId('audit-guard') const mailbox = rpcFor(ownerId) @@ -67,7 +102,7 @@ test('D1 mirror accepts only non-authoritative bounded inbound rejection audits' }) await runInDurableObject(stubFor(ownerId), async (instance) => { - await assertMailboxThrows(/missing-row bootstrap intent/, () => + await assertMailboxThrows(/missing-row bootstrap RPC/, () => instance.upsertDeliveryEvents({ ownerId, events: [ @@ -79,7 +114,7 @@ test('D1 mirror accepts only non-authoritative bounded inbound rejection audits' ], }), ) - await assertMailboxThrows(/missing-row bootstrap intent/, () => + await assertMailboxThrows(/missing-row bootstrap RPC/, () => instance.upsertDeliveryEvents({ ownerId, events: [ @@ -93,7 +128,7 @@ test('D1 mirror accepts only non-authoritative bounded inbound rejection audits' ], }), ) - await assertMailboxThrows(/missing-row bootstrap intent/, () => + await assertMailboxThrows(/missing-row bootstrap RPC/, () => instance.upsertDeliveryEvents({ ownerId, events: [ @@ -143,3 +178,112 @@ test('audit-shaped D1 snapshots cannot overwrite authoritative Mailbox rows', as fingerprint: 'authoritative-fingerprint', }) }) + +test('bootstrapDeliveryEvents is owner-bound, validated, bounded, transactional, and missing-only', async () => { + const ownerId = uniqueUserId('bootstrap-rpc') + const otherOwnerId = uniqueUserId('bootstrap-rpc-other') + const mailbox = rpcFor(ownerId) + const deliveryId = 'email-inbound-delivery:legacy-rpc' + const pending = legacyPendingEvent(ownerId, deliveryId) + + await expect( + mailbox.bootstrapDeliveryEvents({ ownerId, events: [pending] }), + ).resolves.toEqual({ + inserted: 1, + existing: 0, + skipped: 0, + results: [{ eventId: deliveryId, status: 'inserted' }], + }) + await mailbox.claimInboundDeliveryStorage({ + ownerId, + deliveryId, + expectedAttachmentCount: 3, + now: '2026-08-02T13:00:00.000Z', + }) + const newer = await mailbox.getInboundDelivery({ ownerId, deliveryId }) + expect(newer).toMatchObject({ state: 'storing', expectedAttachmentCount: 3 }) + + await expect( + mailbox.bootstrapDeliveryEvents({ ownerId, events: [pending] }), + ).resolves.toMatchObject({ inserted: 0, existing: 1, skipped: 0 }) + expect( + await mailbox.getInboundDelivery({ ownerId, deliveryId }), + ).toMatchObject({ + state: 'storing', + expectedAttachmentCount: 3, + updatedAt: newer?.updatedAt, + }) + const skippedId = 'email-inbound-delivery:provider-id-conflict' + await mailbox.upsertDeliveryEvent({ + ownerId, + event: baseDeliveryEvent({ + id: 'provider-id-conflict-holder', + provider: 'kody', + providerEventId: skippedId, + }), + }) + await expect( + mailbox.bootstrapDeliveryEvents({ + ownerId, + events: [legacyPendingEvent(ownerId, skippedId)], + }), + ).resolves.toEqual({ + inserted: 0, + existing: 0, + skipped: 1, + results: [{ eventId: skippedId, status: 'skipped' }], + }) + + await runInDurableObject(stubFor(ownerId), async (instance) => { + await assertMailboxThrows(/ownerId mismatch/, () => + instance.bootstrapDeliveryEvents({ + ownerId: otherOwnerId, + events: [ + legacyPendingEvent(otherOwnerId, 'email-inbound-delivery:other'), + ], + }), + ) + await assertMailboxThrows( + /requires an inbound lifecycle or dedupe snapshot/, + () => + instance.bootstrapDeliveryEvents({ + ownerId, + events: [baseDeliveryEvent({ id: 'not-inbound', provider: 'kody' })], + }), + ) + await assertMailboxThrows(/valid owner-bound complete snapshot/, () => + instance.bootstrapDeliveryEvents({ + ownerId, + events: [ + legacyPendingEvent(ownerId, 'email-inbound-delivery:txn-valid'), + { + ...legacyPendingEvent( + ownerId, + 'email-inbound-delivery:txn-invalid', + ), + state: 'storing', + }, + ], + }), + ) + await assertMailboxThrows(/exceed max of 100/, () => + instance.bootstrapDeliveryEvents({ + ownerId, + events: Array.from( + { length: mailboxUpsertDeliveryEventsMax + 1 }, + (_, index) => + legacyPendingEvent( + ownerId, + `email-inbound-delivery:overflow-${index}`, + ), + ), + }), + ) + }) + expect( + await mailbox.getInboundDelivery({ + ownerId, + deliveryId: 'email-inbound-delivery:txn-valid', + }), + ).toBeNull() +}) diff --git a/packages/worker/src/email/mailbox-inbound-bootstrap.ts b/packages/worker/src/email/mailbox-inbound-bootstrap.ts index 18f52bad45..7011386583 100644 --- a/packages/worker/src/email/mailbox-inbound-bootstrap.ts +++ b/packages/worker/src/email/mailbox-inbound-bootstrap.ts @@ -70,7 +70,7 @@ function hasNoInboundAuthorityState( ) } -function isPreClaimAuditSnapshot( +export function isPreClaimAuditSnapshot( event: MailboxDeliveryEventInput | MailboxDeliveryEventRecord, ) { if ( @@ -114,6 +114,12 @@ function isPreClaimAuditSnapshot( return event.id === `email-rejections:${event.inboxId}:${detail.day}` } +export function isUserInboundLegacyAuthoritySnapshot( + event: MailboxDeliveryEventInput, +) { + return isUserInboundAuthorityEvent(event) && !isPreClaimAuditSnapshot(event) +} + function readMailboxDeliveryEvent( sql: SqlStorage, eventId: string, @@ -128,37 +134,16 @@ function readMailboxDeliveryEvent( } /** - * Fence the only D1 → Mailbox USER inbound bridge. Bootstrap accepts a - * validated complete snapshot only when the owner-bound row is still missing. + * Validate the only D1 → Mailbox USER inbound bridge. Callers may insert this + * snapshot only when its owner-bound row is missing. */ -export function shouldSkipMailboxDeliveryEventWrite( - sql: SqlStorage, - input: { - ownerId: string - event: MailboxDeliveryEventInput - intent?: 'user-inbound-bootstrap' - hasLatestDeliveryStatus: boolean - }, -) { - const inboundAuthorityEvent = isUserInboundAuthorityEvent(input.event) - if (inboundAuthorityEvent && input.intent !== 'user-inbound-bootstrap') { - if (!isPreClaimAuditSnapshot(input.event)) { - throw new Error( - 'USER inbound delivery events require the missing-row bootstrap intent.', - ) - } - const existing = readMailboxDeliveryEvent(sql, input.event.id) - return existing != null && !isPreClaimAuditSnapshot(existing) - } - if (!inboundAuthorityEvent && input.intent === 'user-inbound-bootstrap') { +export function assertUserInboundLegacyBootstrapSnapshot(input: { + ownerId: string + event: MailboxDeliveryEventInput +}) { + if (!isUserInboundLegacyAuthoritySnapshot(input.event)) { throw new Error( - 'USER inbound bootstrap intent requires an inbound delivery provider.', - ) - } - if (input.intent !== 'user-inbound-bootstrap') return false - if (input.hasLatestDeliveryStatus) { - throw new Error( - 'USER inbound bootstrap cannot update latest delivery status.', + 'USER inbound bootstrap requires an inbound lifecycle or dedupe snapshot.', ) } const delivery = parseStrictInboundDeliveryDetailJson(input.event.detailJson) @@ -166,25 +151,113 @@ export function shouldSkipMailboxDeliveryEventWrite( input.event.provider === mailboxInboundDedupeProvider && delivery ? mailboxInboundDedupePointerId(delivery.fingerprint) : delivery?.deliveryId + const expectedEventType = + delivery?.state === 'received' + ? 'received' + : delivery?.state === 'rejected' + ? 'rejected' + : 'receive_started' + const optionalMatches = (actual: unknown, expected: unknown) => + actual === (expected ?? null) if ( !delivery || delivery.userId !== input.ownerId || delivery.provider !== mailboxInboundProvider || input.event.id !== expectedEventId || + input.event.inboxId !== delivery.inboxId || + input.event.eventType !== expectedEventType || + input.event.providerMessageId != null || + input.event.providerEventId !== input.event.id || + input.event.messageId !== + (delivery.state === 'received' ? delivery.messageId : null) || input.event.state !== delivery.state || - input.event.fingerprint !== delivery.fingerprint + input.event.fingerprint !== delivery.fingerprint || + !optionalMatches(input.event.storageLease, delivery.storageLease) || + !optionalMatches(input.event.storageLeaseAt, delivery.storageLeaseAt) || + !optionalMatches(input.event.cleanupLease, delivery.cleanupLease) || + !optionalMatches(input.event.cleanupLeaseAt, delivery.cleanupLeaseAt) || + !optionalMatches( + input.event.expectedAttachmentCount, + delivery.expectedAttachmentCount, + ) || + !optionalMatches( + input.event.finalizationToken, + delivery.finalizationToken, + ) || + !optionalMatches(input.event.dedupeExpiresAt, delivery.dedupeExpiresAt) || + !optionalMatches( + input.event.usageEffectRecordedAt, + delivery.usageEffectRecordedAt, + ) || + !optionalMatches( + input.event.usageEffectSuppressedAt, + delivery.usageEffectSuppressedAt, + ) || + !optionalMatches(input.event.usageStartedAt, delivery.usageStartedAt) || + !optionalMatches(input.event.usageMonth, delivery.usageMonth) || + !optionalMatches(input.event.usageBytes, delivery.usageBytes) || + !optionalMatches(input.event.usageDurationMs, delivery.usageDurationMs) || + !optionalMatches( + input.event.usageEffectRetryAt, + delivery.usageEffectRetryAt, + ) || + !optionalMatches(input.event.usageEffectLease, delivery.usageEffectLease) || + !optionalMatches( + input.event.usageEffectLeaseAt, + delivery.usageEffectLeaseAt, + ) || + !optionalMatches( + input.event.subscriptionEffectState, + delivery.subscriptionEffectState, + ) || + !optionalMatches( + input.event.subscriptionEffectLease, + delivery.subscriptionEffectLease, + ) || + !optionalMatches( + input.event.subscriptionEffectLeaseAt, + delivery.subscriptionEffectLeaseAt, + ) || + !optionalMatches( + input.event.subscriptionEffectRetryAt, + delivery.subscriptionEffectRetryAt, + ) || + !optionalMatches( + input.event.subscriptionEffectAttemptCount, + delivery.subscriptionEffectAttemptCount, + ) || + !optionalMatches( + input.event.subscriptionEffectDeadLetterAt, + delivery.subscriptionEffectDeadLetterAt, + ) || + !optionalMatches( + input.event.subscriptionEffectLastError, + delivery.subscriptionEffectLastError, + ) ) { throw new Error( 'USER inbound bootstrap requires a valid owner-bound complete snapshot.', ) } +} + +/** + * Reject authority-bearing lifecycle/dedupe snapshots from normal mirror + * upserts. The only inbound-provider exception is the bounded pre-claim audit. + */ +export function shouldSkipMailboxDeliveryEventWrite( + sql: SqlStorage, + input: { + event: MailboxDeliveryEventInput + }, +) { + if (!isUserInboundAuthorityEvent(input.event)) return false + if (!isPreClaimAuditSnapshot(input.event)) { + throw new Error( + 'USER inbound delivery events require the missing-row bootstrap RPC.', + ) + } const eventId = assertMailboxNonEmptyString(input.event.id, 'event.id') - return ( - sql - .exec( - `SELECT id FROM email_delivery_events WHERE id = ? LIMIT 1`, - eventId, - ) - .toArray().length > 0 - ) + const existing = readMailboxDeliveryEvent(sql, eventId) + return existing != null && !isPreClaimAuditSnapshot(existing) } diff --git a/packages/worker/src/email/mailbox-mirror.node.test.ts b/packages/worker/src/email/mailbox-mirror.node.test.ts index 23ae74b51b..b12b9671e2 100644 --- a/packages/worker/src/email/mailbox-mirror.node.test.ts +++ b/packages/worker/src/email/mailbox-mirror.node.test.ts @@ -550,6 +550,87 @@ test('mailbox mirror batch helper uses one RPC, one telemetry outcome, and maps expect(upsertDeliveryEvents).toHaveBeenCalledTimes(1) }) +test('mailbox mirror batch partitions legacy authority from audits and counts existing/skipped without errors', async () => { + const legacyExisting = baseDeliveryEventInput({ + id: 'email-inbound-delivery:existing', + provider: 'cloudflare-email-routing', + eventType: 'receive_started', + state: 'pending', + fingerprint: 'existing-fingerprint', + }) + const legacySkipped = baseDeliveryEventInput({ + id: 'email-inbound-dedupe:skipped-fingerprint', + provider: 'cloudflare-email-routing-dedupe', + eventType: 'receive_started', + state: 'pending', + fingerprint: 'skipped-fingerprint', + }) + const audit = baseDeliveryEventInput({ + id: 'email-rejections:inbox-audit:2026-07-02', + messageId: null, + inboxId: 'inbox-audit', + provider: 'cloudflare-email-routing', + eventType: 'rejected', + detailJson: JSON.stringify({ + aggregate: true, + day: '2026-07-02', + count: 1, + last_reason: 'Recipient mailbox is over quota.', + last_phase: 'entitlement', + last_at: '2026-07-02T10:00:00.000Z', + }), + }) + const upsertDeliveryEvents = vi.fn(async () => ({ + results: [{ eventId: audit.id, inserted: true, accepted: true }], + })) + const bootstrapDeliveryEvents = vi.fn(async () => ({ + inserted: 0, + existing: 1, + skipped: 1, + results: [ + { eventId: legacyExisting.id, status: 'existing' as const }, + { eventId: legacySkipped.id, status: 'skipped' as const }, + ], + })) + const { env, writeDataPoint } = fakeMailboxEnv({ + upsertDeliveryEvents, + bootstrapDeliveryEvents, + }) + + await expect( + mirrorMailboxDeliveryEventSnapshots({ + env, + ownerId: 'user-aaa', + events: [legacyExisting, audit, legacySkipped], + }), + ).resolves.toEqual([ + { eventId: legacyExisting.id, result: { status: 'stale' } }, + { eventId: audit.id, result: { status: 'mirrored' } }, + { + eventId: legacySkipped.id, + result: { status: 'skipped', reason: 'user-inbound-authority' }, + }, + ]) + expect(upsertDeliveryEvents).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + events: [audit], + }) + expect(bootstrapDeliveryEvents).toHaveBeenCalledWith({ + ownerId: 'user-aaa', + events: [legacyExisting, legacySkipped], + }) + expect(consoleWarn).not.toHaveBeenCalled() + expect(writeDataPoint).toHaveBeenCalledWith( + expect.objectContaining({ + blobs: [ + 'mailbox_mirror:upsert_delivery_event_batch', + 'skipped', + expect.any(String), + ], + }), + ) +}) + test('mailbox mirror batch helper maps timeout and error to every event', async () => { vi.useFakeTimers() consoleWarn.mockImplementation(() => {}) diff --git a/packages/worker/src/email/mailbox-mirror.ts b/packages/worker/src/email/mailbox-mirror.ts index d6329e12a1..200b904ace 100644 --- a/packages/worker/src/email/mailbox-mirror.ts +++ b/packages/worker/src/email/mailbox-mirror.ts @@ -15,6 +15,7 @@ import { toMailboxMessageInput, toMailboxThreadInput, } from './mailbox-snapshots.ts' +import { isUserInboundLegacyAuthoritySnapshot } from './mailbox-inbound-bootstrap.ts' import { type EmailAttachmentRecord, type EmailDeliveryStatus, @@ -31,7 +32,6 @@ import { type MailboxSetMessageClassificationInput, type MailboxTouchThreadInput, type MailboxUpdateMessageDeliveryInput, - type MailboxUpsertDeliveryEventsResult, } from './mailbox-types.ts' /** @@ -81,7 +81,10 @@ function resolveMirrorOwner( ownerId: string | null | undefined, ): { ok: true; ownerId: string } | { ok: false; result: MailboxMirrorResult } { if (ownerId == null || ownerId.length === 0) { - return { ok: false, result: { status: 'skipped', reason: 'missing-owner' } } + return { + ok: false, + result: { status: 'skipped', reason: 'missing-owner' }, + } } if (isSystemEmailOwner(ownerId)) { return { @@ -178,12 +181,20 @@ function outcomeFromAccepted(accepted: boolean): MailboxMirrorResult { return accepted ? { status: 'mirrored' } : { status: 'stale' } } -function outcomeFromBatchAccepted( - results: MailboxUpsertDeliveryEventsResult['results'], +function aggregateBatchOutcome( + results: Array, ): MailboxMirrorResult { - return results.every((item) => item.accepted) - ? { status: 'mirrored' } - : { status: 'stale' } + for (const status of [ + 'error', + 'timeout', + 'skipped', + 'missing', + 'stale', + ] as const) { + const result = results.find((item) => item.result.status === status)?.result + if (result) return result + } + return { status: 'mirrored' } } /** @@ -325,10 +336,11 @@ export async function mirrorMailboxDeliveryEventSnapshot(input: { /** * Best-effort batch delivery-event snapshot mirror. * - * One timeout (default {@link mailboxMirrorRpcTimeoutMs}) and one - * `upsert_delivery_event_batch` telemetry outcome. Timeout/error/skip apply - * uniformly to each per-event summary entry. Empty `events` returns `[]` - * without RPC or telemetry. + * Partitions legacy USER inbound authority snapshots to missing-only bootstrap + * and all other rows to normal upsert. Each non-empty subset gets the timeout + * bound (default {@link mailboxMirrorRpcTimeoutMs}); one aggregate + * `upsert_delivery_event_batch` telemetry outcome covers the input page. Empty + * `events` returns `[]` without RPC or telemetry. */ export async function mirrorMailboxDeliveryEventSnapshots(input: { env: MailboxMirrorEnv @@ -351,46 +363,121 @@ export async function mirrorMailboxDeliveryEventSnapshots(input: { return mapUniformBatchResults(input.events, skippedMailbox) } - try { - const raced = await awaitMailboxMirrorRpc( - mailboxRpc({ - env: input.env, - userId: owner.ownerId, - }).upsertDeliveryEvents({ - ownerId: owner.ownerId, - events: input.events, - }), - input.timeoutMs ?? mailboxMirrorRpcTimeoutMs, - ) - if (!raced.ok) { - const result = { status: 'timeout' as const } - recordMirrorOutcome(input.env, owner.ownerId, operation, result) - return mapUniformBatchResults(input.events, result) + const timeoutMs = input.timeoutMs ?? mailboxMirrorRpcTimeoutMs + const rpc = mailboxRpc({ + env: input.env, + userId: owner.ownerId, + }) + const bootstrapEvents = input.events.filter( + isUserInboundLegacyAuthoritySnapshot, + ) + const normalEvents = input.events.filter( + (event) => !isUserInboundLegacyAuthoritySnapshot(event), + ) + const resultsByEventId = new Map() + + if (normalEvents.length > 0) { + try { + const raced = await awaitMailboxMirrorRpc( + rpc.upsertDeliveryEvents({ + ownerId: owner.ownerId, + events: normalEvents, + }), + timeoutMs, + ) + if (!raced.ok) { + for (const event of normalEvents) { + resultsByEventId.set(event.id, { status: 'timeout' }) + } + } else { + const byEventId = new Map( + raced.value.results.map((item) => [item.eventId, item] as const), + ) + for (const event of normalEvents) { + const item = byEventId.get(event.id) + resultsByEventId.set( + event.id, + item ? outcomeFromAccepted(item.accepted) : { status: 'missing' }, + ) + } + } + } catch (error) { + console.warn('mailbox-mirror-delivery-event-batch-failed', error) + for (const event of normalEvents) { + resultsByEventId.set(event.id, { status: 'error', error }) + } } - recordMirrorOutcome( - input.env, - owner.ownerId, - operation, - outcomeFromBatchAccepted(raced.value.results), - ) - const byEventId = new Map( - raced.value.results.map((item) => [item.eventId, item] as const), - ) - return input.events.map((event) => { - const item = byEventId.get(event.id) - return { - eventId: event.id, - result: item - ? outcomeFromAccepted(item.accepted) - : { status: 'missing' as const }, + } + + if (bootstrapEvents.length > 0) { + try { + const raced = await awaitMailboxMirrorRpc( + rpc.bootstrapDeliveryEvents({ + ownerId: owner.ownerId, + events: bootstrapEvents, + }), + timeoutMs, + ) + if (!raced.ok) { + for (const event of bootstrapEvents) { + resultsByEventId.set(event.id, { status: 'timeout' }) + } + } else { + const byEventId = new Map( + raced.value.results.map((item) => [item.eventId, item] as const), + ) + for (const event of bootstrapEvents) { + const item = byEventId.get(event.id) + const status = item?.status + let result: MailboxMirrorResult + switch (status) { + case 'inserted': + result = { status: 'mirrored' } + break + case 'existing': + result = { status: 'stale' } + break + case 'skipped': + result = { + status: 'skipped', + reason: 'user-inbound-authority', + } + break + case undefined: + result = { status: 'missing' } + break + default: { + const exhaustive: never = status + throw new Error( + `Unhandled bootstrap delivery-event status: ${String(exhaustive)}`, + ) + } + } + resultsByEventId.set(event.id, result) + } } - }) - } catch (error) { - console.warn('mailbox-mirror-delivery-event-batch-failed', error) - const result = { status: 'error' as const, error } - recordMirrorOutcome(input.env, owner.ownerId, operation, result) - return mapUniformBatchResults(input.events, result) + } catch (error) { + console.warn( + 'mailbox-mirror-delivery-event-bootstrap-batch-failed', + error, + ) + for (const event of bootstrapEvents) { + resultsByEventId.set(event.id, { status: 'error', error }) + } + } } + + const results = input.events.map((event) => ({ + eventId: event.id, + result: resultsByEventId.get(event.id) ?? ({ status: 'missing' } as const), + })) + recordMirrorOutcome( + input.env, + owner.ownerId, + operation, + aggregateBatchOutcome(results), + ) + return results } /** Best-effort thread touch mirror. */ diff --git a/packages/worker/src/email/mailbox-parity-phases.ts b/packages/worker/src/email/mailbox-parity-phases.ts index fc68c25383..c831605baf 100644 --- a/packages/worker/src/email/mailbox-parity-phases.ts +++ b/packages/worker/src/email/mailbox-parity-phases.ts @@ -19,7 +19,7 @@ import { mailboxUpsertDeliveryEventsMax } from './mailbox-types.ts' export const mailboxParityMessagePageSize = 10 /** - * Events per backfill page / single `upsertDeliveryEvents` RPC. + * Events per backfill page, partitioned across normal/bootstrap batch RPCs. * Must stay ≤ {@link mailboxUpsertDeliveryEventsMax}. */ export const mailboxParityEventPageSize = 25 @@ -238,8 +238,8 @@ export async function backfillMessagesForUser(input: { } /** - * Keyset-page ready delivery-event snapshots and mirror each page with one - * `upsertDeliveryEvents` batch RPC ({@link mailboxParityEventMirrorTimeoutMs}). + * Keyset-page ready delivery-event snapshots and mirror each page with bounded + * normal/bootstrap batch RPCs ({@link mailboxParityEventMirrorTimeoutMs}). * * Cursor advances through per-event mirrored/stale/missing results in page * order (including equal `createdAt` by id). Uniform timeout/error/skip keep diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts index bdd9548ecc..8331c9ba2b 100644 --- a/packages/worker/src/email/mailbox-reconcile.workers.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -3,6 +3,7 @@ import { expect, test, vi } from 'vitest' import { silenceIncidentalRuntimeWarnings } from '#worker/test-support/incidental-runtime-warnings.ts' import { createStableUserIdFromEmail } from '#worker/user-id.ts' import { ensureUsersTestSchema } from '#worker/users-test-schema.ts' +import { emailRawMimeKey } from './blob-keys.ts' import { mailboxLiveMirrorMaxEvents } from './mailbox-live-mirror.ts' import { countD1MailboxParity, @@ -131,6 +132,105 @@ async function seedPreClaimAuditEvent(input: { return eventId } +async function seedLegacyInboundEvent(input: { + userId: string + deliveryId: string + state: 'pending' | 'storing' | 'received' + createdAt: string + messageId?: string + fingerprint?: string + provider?: 'cloudflare-email-routing' | 'cloudflare-email-routing-dedupe' + eventId?: string +}) { + const provider = input.provider ?? 'cloudflare-email-routing' + const fingerprint = input.fingerprint ?? `fingerprint-${input.deliveryId}` + const messageId = + input.messageId ?? `email-inbound-message:${input.deliveryId}` + const eventId = + input.eventId ?? + (provider === 'cloudflare-email-routing-dedupe' + ? `email-inbound-dedupe:${fingerprint}` + : input.deliveryId) + const detail = { + fingerprint, + deliveryId: input.deliveryId, + messageId, + threadId: `email-inbound-thread:${input.deliveryId}`, + rawMimeKey: emailRawMimeKey(input.userId, messageId), + userId: input.userId, + inboxId: 'legacy-inbox', + recipient: 'owner@example.test', + envelopeFrom: 'sender@example.test', + provider: 'cloudflare-email-routing', + quotaDay: '2026-07-01', + dedupeExpiresAt: '2026-09-01T00:00:00.000Z', + state: input.state, + ...(input.state === 'storing' + ? { + storageLease: 'legacy-storage-lease', + storageLeaseAt: '2026-07-01T12:01:30.000Z', + expectedAttachmentCount: 2, + } + : {}), + ...(input.state === 'received' + ? { + expectedAttachmentCount: 1, + finalizationToken: 'legacy-finalization-token', + usageEffectRecordedAt: '2026-07-01T12:04:00.000Z', + usageStartedAt: '2026-07-01T12:02:30.000Z', + usageMonth: '2026-07', + usageBytes: 4096, + usageDurationMs: 900, + subscriptionEffectState: 'complete', + } + : {}), + } + await env.APP_DB.prepare( + `INSERT INTO email_delivery_events ( + id, message_id, user_id, inbox_id, event_type, provider, + provider_event_id, detail_json, needs_effect_reconcile, + state, fingerprint, storage_lease, storage_lease_at, + expected_attachment_count, finalization_token, dedupe_expires_at, + usage_effect_recorded_at, usage_started_at, usage_month, usage_bytes, + usage_duration_ms, subscription_effect_state, created_at, updated_at + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + )`, + ) + .bind( + eventId, + input.state === 'received' ? messageId : null, + input.userId, + detail.inboxId, + input.state === 'received' ? 'received' : 'receive_started', + provider, + eventId, + JSON.stringify(detail), + 0, + input.state, + fingerprint, + 'storageLease' in detail ? detail.storageLease : null, + 'storageLeaseAt' in detail ? detail.storageLeaseAt : null, + 'expectedAttachmentCount' in detail + ? detail.expectedAttachmentCount + : null, + 'finalizationToken' in detail ? detail.finalizationToken : null, + detail.dedupeExpiresAt, + 'usageEffectRecordedAt' in detail ? detail.usageEffectRecordedAt : null, + 'usageStartedAt' in detail ? detail.usageStartedAt : null, + 'usageMonth' in detail ? detail.usageMonth : null, + 'usageBytes' in detail ? detail.usageBytes : null, + 'usageDurationMs' in detail ? detail.usageDurationMs : null, + 'subscriptionEffectState' in detail + ? detail.subscriptionEffectState + : null, + input.createdAt, + input.createdAt, + ) + .run() + return { eventId, fingerprint } +} + async function readParityState(userId: string) { return env.APP_DB.prepare( `SELECT @@ -522,3 +622,186 @@ test('reconcileMailboxParity eventually repairs >100 bound delivery events', asy .deliveryEvents, ).toBe(heavyEventCount) }, 60_000) + +test('reconcileMailboxParity bootstraps production legacy USER inbound shapes without overwriting authority', async () => { + silenceIncidentalRuntimeWarnings() + await ensureUsersTestSchema({ db: env.APP_DB }) + await ensureEmailTestSchema(env.APP_DB) + await env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = '9999-12-31T23:59:59.999Z'`, + ).run() + + const userId = await seedParityUser({ + email: `legacy-inbound-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + const receivedMessageId = 'legacy-received-message' + await seedMessageGraph({ + userId, + messageId: receivedMessageId, + threadId: 'legacy-received-thread', + createdAt: '2026-07-01T12:02:00.000Z', + eventCount: 0, + }) + const pendingId = 'email-inbound-delivery:legacy-pending' + const storingId = 'email-inbound-delivery:legacy-storing' + const receivedId = 'email-inbound-delivery:legacy-received' + const pending = await seedLegacyInboundEvent({ + userId, + deliveryId: pendingId, + state: 'pending', + createdAt: '2026-07-01T12:00:00.000Z', + }) + await seedLegacyInboundEvent({ + userId, + deliveryId: storingId, + state: 'storing', + createdAt: '2026-07-01T12:01:00.000Z', + }) + await seedLegacyInboundEvent({ + userId, + deliveryId: receivedId, + messageId: receivedMessageId, + state: 'received', + createdAt: '2026-07-01T12:03:00.000Z', + }) + await seedLegacyInboundEvent({ + userId, + deliveryId: pendingId, + fingerprint: pending.fingerprint, + state: 'pending', + provider: 'cloudflare-email-routing-dedupe', + createdAt: '2026-07-01T12:00:01.000Z', + }) + const auditEventId = await seedPreClaimAuditEvent({ + userId, + inboxId: 'legacy-inbox', + createdAt: '2026-07-01T12:05:00.000Z', + }) + + const mailbox = rpcFor(userId) + expect(await mailbox.countMailbox()).toEqual({ + threads: 0, + messages: 0, + attachments: 0, + deliveryEvents: 0, + }) + const firstNow = new Date('2026-08-02T15:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), + ).resolves.toMatchObject({ + scanned: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) + expect( + await mailbox.getInboundDelivery({ + ownerId: userId, + deliveryId: pendingId, + }), + ).toMatchObject({ state: 'pending' }) + expect( + await mailbox.getInboundDelivery({ + ownerId: userId, + deliveryId: storingId, + }), + ).toMatchObject({ + state: 'storing', + storageLease: 'legacy-storage-lease', + expectedAttachmentCount: 2, + }) + expect( + await mailbox.getInboundDelivery({ + ownerId: userId, + deliveryId: receivedId, + }), + ).toMatchObject({ + state: 'received', + finalizationToken: 'legacy-finalization-token', + usageEffectRecordedAt: '2026-07-01T12:04:00.000Z', + subscriptionEffectState: 'complete', + }) + expect( + await mailbox.getInboundDeliveryWindow({ + ownerId: userId, + fingerprint: pending.fingerprint, + now: '2026-08-02T15:00:00.000Z', + }), + ).toMatchObject({ deliveryId: pendingId }) + expect( + (await mailbox.listDeliveryEvents({ messageId: null, limit: 20 })).some( + (event) => event.id === auditEventId, + ), + ).toBe(true) + + const replayEventBackfill = () => + env.APP_DB.prepare( + `UPDATE users + SET mailbox_parity_checked_at = NULL, + mailbox_parity_event_backfill_cursor_created_at = NULL, + mailbox_parity_event_backfill_cursor_id = NULL, + mailbox_parity_event_backfill_completed_at = NULL + WHERE stable_user_id = ?`, + ) + .bind(userId) + .run() + const countAfterFirst = await mailbox.countMailbox() + await replayEventBackfill() + const secondNow = new Date('2026-08-02T15:30:00.000Z') + await expect( + reconcileMailboxParity({ env, now: secondNow, batchSize: 1 }), + ).resolves.toMatchObject({ + scanned: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(await mailbox.countMailbox()).toEqual(countAfterFirst) + + await mailbox.claimInboundDeliveryStorage({ + ownerId: userId, + deliveryId: pendingId, + expectedAttachmentCount: 7, + now: '2026-08-02T15:01:00.000Z', + }) + const newerAuthority = await mailbox.getInboundDelivery({ + ownerId: userId, + deliveryId: pendingId, + }) + expect(newerAuthority).toMatchObject({ + state: 'storing', + expectedAttachmentCount: 7, + }) + await replayEventBackfill() + + const thirdNow = new Date('2026-08-02T16:00:00.000Z') + await expect( + reconcileMailboxParity({ env, now: thirdNow, batchSize: 1 }), + ).resolves.toMatchObject({ + scanned: 1, + compared: 1, + matched: 1, + mismatched: 0, + failed: 0, + }) + expect(await mailbox.countMailbox()).toEqual( + await countD1MailboxParity({ db: env.APP_DB, userId }), + ) + expect( + await mailbox.getInboundDelivery({ + ownerId: userId, + deliveryId: pendingId, + }), + ).toMatchObject({ + state: 'storing', + expectedAttachmentCount: 7, + updatedAt: newerAuthority?.updatedAt, + }) +}, 30_000) diff --git a/packages/worker/src/email/mailbox-types.ts b/packages/worker/src/email/mailbox-types.ts index aa4f4683cc..8ab2c351f8 100644 --- a/packages/worker/src/email/mailbox-types.ts +++ b/packages/worker/src/email/mailbox-types.ts @@ -478,6 +478,18 @@ export type MailboxUpsertDeliveryEventsResult = { results: Array } +export type MailboxBootstrapDeliveryEventItemResult = { + eventId: string + status: 'inserted' | 'existing' | 'skipped' +} + +export type MailboxBootstrapDeliveryEventsResult = { + inserted: number + existing: number + skipped: number + results: Array +} + /** * Mirror / read / retention / purge surface. Authoritative USER inbound ledger * CAS RPCs are intersected below; `system:email` remains D1-only. @@ -499,14 +511,12 @@ type MailboxCoreRpc = { attachments?: Array }) => Promise<{ ok: true; accepted: boolean }> /** - * Complete delivery-event snapshot upsert. USER inbound only uses this for - * the one-time pre-deploy D1 bootstrap bridge. + * Complete delivery-event snapshot upsert. Rejects USER inbound + * lifecycle/dedupe authority snapshots; use `bootstrapDeliveryEvents`. */ upsertDeliveryEvent: (input: { ownerId: string event: MailboxDeliveryEventInput - /** Missing-row-only bridge for pre-deploy USER inbound D1 snapshots. */ - intent?: 'user-inbound-bootstrap' latestDeliveryStatus?: { messageId: string deliveryStatus: EmailDeliveryStatus @@ -526,6 +536,14 @@ type MailboxCoreRpc = { ownerId: string events: Array }) => Promise + /** + * Missing-row-only deployment bridge for validated legacy USER inbound + * lifecycle/dedupe snapshots. Existing rows are never updated. + */ + bootstrapDeliveryEvents: (input: { + ownerId: string + events: Array + }) => Promise touchThread: ( input: MailboxTouchThreadInput, ) => Promise From cfe74bf39178893553b0302da685162c4ee22925 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 2 Aug 2026 19:01:38 +0000 Subject: [PATCH 62/62] fix(email): preserve legacy reconcile schedules Co-authored-by: Kent C. Dodds --- .../contributing/architecture/data-storage.md | 16 ++- .../src/email/inbound-delivery-projection.ts | 25 ++-- packages/worker/src/email/inbound-delivery.ts | 20 +++ .../email/mailbox-delivery-event-bootstrap.ts | 12 +- ...ox-inbound-authority-guard.workers.test.ts | 22 +++ .../src/email/mailbox-inbound-bootstrap.ts | 41 +++++- .../worker/src/email/mailbox-parity-phases.ts | 3 +- .../email/mailbox-reconcile.workers.test.ts | 129 +++++++++++++++--- 8 files changed, 230 insertions(+), 38 deletions(-) diff --git a/docs/contributing/architecture/data-storage.md b/docs/contributing/architecture/data-storage.md index 5675f9f19d..0038ac6a83 100644 --- a/docs/contributing/architecture/data-storage.md +++ b/docs/contributing/architecture/data-storage.md @@ -746,8 +746,13 @@ non-inbound events continue through normal `upsertDeliveryEvents`. The bootstrap RPC accepts at most 100 snapshots, validates owner/provider/detail coherence, and reports inserted/existing/skipped counts. It never updates an existing Mailbox row. Normal delivery-event upserts continue rejecting USER inbound -authority snapshots. `system:email` stays on the existing D1 implementation and -cannot bootstrap a Mailbox. Migration +authority snapshots. Legacy lifecycle bootstrap preserves canonical +`reconcileAfter` and orphan-cleaned `cleanupRetryAt` schedules in Mailbox +columns/detail JSON so due work cannot run early; irrelevant dedupe/terminal +schedule columns remain null. A malformed schedule is skipped per row, allowing +the normal audit subset to commit; parity records a count mismatch instead of +treating the mixed page as an RPC failure. `system:email` stays on the existing +D1 implementation and cannot bootstrap a Mailbox. Migration `0129-email-inbound-mailbox-authority-mirror.sql` is additive: it promotes compatibility/fence fields and adds the cross-store usage effect idempotency ledger; it drops no tables. Destructive follow-up work remains gated on the @@ -1011,9 +1016,10 @@ Per user, the lane: RPCs repeatedly timed out on a lagging owner and prevented soak under the 10s lane budget; page batches restore convergence while live dual-write keeps the 1s bound. Cursor advances through per-event mirrored/stale/missing results - (equal `created_at` progresses by id); uniform timeout/error/unconfigured - retains the cursor so the next tick reloads the same page. Rows deleted - before the snapshot load simply do not appear. + and USER-inbound bootstrap skips (so count comparison exposes malformed + legacy rows); equal `created_at` progresses by id. Uniform + timeout/error/unconfigured retains the cursor so the next tick reloads the + same page. Rows deleted before the snapshot load simply do not appear. 4. **Durable content watermark replay** — after both creation phases complete, opens a frozen window `(watermark, upper]` (`mailbox_parity_content_replay_upper_at` set once when the window opens; diff --git a/packages/worker/src/email/inbound-delivery-projection.ts b/packages/worker/src/email/inbound-delivery-projection.ts index 5c1262134c..b85f064fe7 100644 --- a/packages/worker/src/email/inbound-delivery-projection.ts +++ b/packages/worker/src/email/inbound-delivery-projection.ts @@ -29,13 +29,7 @@ export function toUserInboundDelivery( userId: string, snapshot: MailboxInboundDeliverySnapshot, ): InboundDelivery { - const { - createdAt: _createdAt, - updatedAt: _updatedAt, - cleanupRetryAt: _cleanupRetryAt, - reconcileAfter: _reconcileAfter, - ...delivery - } = snapshot + const { createdAt: _createdAt, updatedAt: _updatedAt, ...delivery } = snapshot return { ...delivery, userId } } @@ -65,6 +59,19 @@ function toMailboxBootstrapEvent(input: { updatedAt: string }): MailboxDeliveryEventInput { const delivery = input.delivery + const isLifecycle = input.provider === mailboxInboundProvider + const cleanupRetryAt = + isLifecycle && delivery.state === 'orphan-cleaned' + ? (delivery.cleanupRetryAt ?? null) + : null + const reconcileAfter = + isLifecycle && + (delivery.state === 'pending' || + delivery.state === 'storing' || + delivery.state === 'cleaning' || + delivery.state === 'orphan-cleaned') + ? (delivery.reconcileAfter ?? null) + : null return { id: input.eventId, messageId: delivery.state === 'received' ? delivery.messageId : null, @@ -82,10 +89,10 @@ function toMailboxBootstrapEvent(input: { storageLeaseAt: delivery.storageLeaseAt ?? null, cleanupLease: delivery.cleanupLease ?? null, cleanupLeaseAt: delivery.cleanupLeaseAt ?? null, - cleanupRetryAt: null, + cleanupRetryAt, expectedAttachmentCount: delivery.expectedAttachmentCount ?? null, finalizationToken: delivery.finalizationToken ?? null, - reconcileAfter: null, + reconcileAfter, dedupeExpiresAt: delivery.dedupeExpiresAt, usageEffectRecordedAt: delivery.usageEffectRecordedAt ?? null, usageEffectSuppressedAt: delivery.usageEffectSuppressedAt ?? null, diff --git a/packages/worker/src/email/inbound-delivery.ts b/packages/worker/src/email/inbound-delivery.ts index b5ccc86665..ffd5f98410 100644 --- a/packages/worker/src/email/inbound-delivery.ts +++ b/packages/worker/src/email/inbound-delivery.ts @@ -55,7 +55,9 @@ export type InboundDelivery = { expectedAttachmentCount?: number cleanupLease?: string cleanupLeaseAt?: string + cleanupRetryAt?: string finalizationToken?: string + reconcileAfter?: string usageEffectRecordedAt?: string usageEffectSuppressedAt?: string usageStartedAt?: string @@ -217,9 +219,15 @@ export function parseInboundDeliveryDetailJson( ...(typeof detail.cleanupLeaseAt === 'string' ? { cleanupLeaseAt: detail.cleanupLeaseAt } : {}), + ...(typeof detail.cleanupRetryAt === 'string' + ? { cleanupRetryAt: detail.cleanupRetryAt } + : {}), ...(typeof detail.finalizationToken === 'string' ? { finalizationToken: detail.finalizationToken } : {}), + ...(typeof detail.reconcileAfter === 'string' + ? { reconcileAfter: detail.reconcileAfter } + : {}), ...(typeof detail.usageEffectRecordedAt === 'string' ? { usageEffectRecordedAt: detail.usageEffectRecordedAt } : {}), @@ -298,6 +306,18 @@ export function parseStrictInboundDeliveryDetailJson( ) { return null } + const record = raw as Record + for (const field of ['cleanupRetryAt', 'reconcileAfter'] as const) { + const value = record[field] + if ( + field in record && + (typeof value !== 'string' || + !Number.isFinite(Date.parse(value)) || + new Date(value).toISOString() !== value) + ) { + return null + } + } return delivery } catch { return null diff --git a/packages/worker/src/email/mailbox-delivery-event-bootstrap.ts b/packages/worker/src/email/mailbox-delivery-event-bootstrap.ts index 40572dac5c..68765cc88b 100644 --- a/packages/worker/src/email/mailbox-delivery-event-bootstrap.ts +++ b/packages/worker/src/email/mailbox-delivery-event-bootstrap.ts @@ -1,5 +1,8 @@ import { writeMailboxDeliveryEventRow } from './mailbox-delivery-events.ts' -import { assertUserInboundLegacyBootstrapSnapshot } from './mailbox-inbound-bootstrap.ts' +import { + assertUserInboundLegacyBootstrapSnapshot, + hasMalformedUserInboundBootstrapSchedule, +} from './mailbox-inbound-bootstrap.ts' import { assertMailboxNonEmptyString, mailboxUpsertDeliveryEventsMax, @@ -33,11 +36,16 @@ export function bootstrapMailboxDeliveryEvents( results: [], } for (const event of input.events) { + const eventId = assertMailboxNonEmptyString(event.id, 'event.id') + if (hasMalformedUserInboundBootstrapSchedule(event)) { + result.skipped += 1 + result.results.push({ eventId, status: 'skipped' }) + continue + } assertUserInboundLegacyBootstrapSnapshot({ ownerId: input.ownerId, event, }) - const eventId = assertMailboxNonEmptyString(event.id, 'event.id') const existing = sql .exec( `SELECT id FROM email_delivery_events WHERE id = ? LIMIT 1`, diff --git a/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts b/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts index ec4ab8686f..3527de7dde 100644 --- a/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts +++ b/packages/worker/src/email/mailbox-inbound-authority-guard.workers.test.ts @@ -233,6 +233,28 @@ test('bootstrapDeliveryEvents is owner-bound, validated, bounded, transactional, skipped: 1, results: [{ eventId: skippedId, status: 'skipped' }], }) + const malformedId = 'email-inbound-delivery:malformed-schedule' + const malformed = legacyPendingEvent(ownerId, malformedId) + await expect( + mailbox.bootstrapDeliveryEvents({ + ownerId, + events: [ + { + ...malformed, + reconcileAfter: 'not-a-timestamp', + detailJson: JSON.stringify({ + ...JSON.parse(malformed.detailJson), + reconcileAfter: 'not-a-timestamp', + }), + }, + ], + }), + ).resolves.toEqual({ + inserted: 0, + existing: 0, + skipped: 1, + results: [{ eventId: malformedId, status: 'skipped' }], + }) await runInDurableObject(stubFor(ownerId), async (instance) => { await assertMailboxThrows(/ownerId mismatch/, () => diff --git a/packages/worker/src/email/mailbox-inbound-bootstrap.ts b/packages/worker/src/email/mailbox-inbound-bootstrap.ts index 7011386583..9d2d5ada0f 100644 --- a/packages/worker/src/email/mailbox-inbound-bootstrap.ts +++ b/packages/worker/src/email/mailbox-inbound-bootstrap.ts @@ -3,7 +3,10 @@ import { mailboxInboundDedupeProvider, mailboxInboundProvider, } from './mailbox-inbound-ledger.ts' -import { parseStrictInboundDeliveryDetailJson } from './inbound-delivery.ts' +import { + parseInboundDeliveryDetailJson, + parseStrictInboundDeliveryDetailJson, +} from './inbound-delivery.ts' import { mapMailboxDeliveryEventRow } from './mailbox-mappers.ts' import { assertMailboxNonEmptyString, @@ -120,6 +123,27 @@ export function isUserInboundLegacyAuthoritySnapshot( return isUserInboundAuthorityEvent(event) && !isPreClaimAuditSnapshot(event) } +export function hasMalformedUserInboundBootstrapSchedule( + event: MailboxDeliveryEventInput, +) { + if (!isUserInboundLegacyAuthoritySnapshot(event)) return false + if (!parseInboundDeliveryDetailJson(event.detailJson)) return false + try { + const detail = JSON.parse(event.detailJson) as Record + return ['cleanupRetryAt', 'reconcileAfter'].some((field) => { + if (!(field in detail)) return false + const value = detail[field] + return ( + typeof value !== 'string' || + !Number.isFinite(Date.parse(value)) || + new Date(value).toISOString() !== value + ) + }) + } catch { + return false + } +} + function readMailboxDeliveryEvent( sql: SqlStorage, eventId: string, @@ -159,6 +183,19 @@ export function assertUserInboundLegacyBootstrapSnapshot(input: { : 'receive_started' const optionalMatches = (actual: unknown, expected: unknown) => actual === (expected ?? null) + const lifecycle = input.event.provider === mailboxInboundProvider + const expectedCleanupRetryAt = + lifecycle && delivery?.state === 'orphan-cleaned' + ? delivery.cleanupRetryAt + : null + const expectedReconcileAfter = + lifecycle && + (delivery?.state === 'pending' || + delivery?.state === 'storing' || + delivery?.state === 'cleaning' || + delivery?.state === 'orphan-cleaned') + ? delivery.reconcileAfter + : null if ( !delivery || delivery.userId !== input.ownerId || @@ -176,6 +213,7 @@ export function assertUserInboundLegacyBootstrapSnapshot(input: { !optionalMatches(input.event.storageLeaseAt, delivery.storageLeaseAt) || !optionalMatches(input.event.cleanupLease, delivery.cleanupLease) || !optionalMatches(input.event.cleanupLeaseAt, delivery.cleanupLeaseAt) || + !optionalMatches(input.event.cleanupRetryAt, expectedCleanupRetryAt) || !optionalMatches( input.event.expectedAttachmentCount, delivery.expectedAttachmentCount, @@ -184,6 +222,7 @@ export function assertUserInboundLegacyBootstrapSnapshot(input: { input.event.finalizationToken, delivery.finalizationToken, ) || + !optionalMatches(input.event.reconcileAfter, expectedReconcileAfter) || !optionalMatches(input.event.dedupeExpiresAt, delivery.dedupeExpiresAt) || !optionalMatches( input.event.usageEffectRecordedAt, diff --git a/packages/worker/src/email/mailbox-parity-phases.ts b/packages/worker/src/email/mailbox-parity-phases.ts index c831605baf..10c390a050 100644 --- a/packages/worker/src/email/mailbox-parity-phases.ts +++ b/packages/worker/src/email/mailbox-parity-phases.ts @@ -133,9 +133,10 @@ export function eventMirrorAllowsCursorAdvance( case 'stale': case 'missing': return true + case 'skipped': + return result.reason === 'user-inbound-authority' case 'timeout': case 'error': - case 'skipped': return false default: { const exhaustive: never = result diff --git a/packages/worker/src/email/mailbox-reconcile.workers.test.ts b/packages/worker/src/email/mailbox-reconcile.workers.test.ts index 8331c9ba2b..fb5670ca8b 100644 --- a/packages/worker/src/email/mailbox-reconcile.workers.test.ts +++ b/packages/worker/src/email/mailbox-reconcile.workers.test.ts @@ -135,12 +135,14 @@ async function seedPreClaimAuditEvent(input: { async function seedLegacyInboundEvent(input: { userId: string deliveryId: string - state: 'pending' | 'storing' | 'received' + state: 'pending' | 'storing' | 'received' | 'orphan-cleaned' createdAt: string messageId?: string fingerprint?: string provider?: 'cloudflare-email-routing' | 'cloudflare-email-routing-dedupe' eventId?: string + cleanupRetryAt?: string + reconcileAfter?: string }) { const provider = input.provider ?? 'cloudflare-email-routing' const fingerprint = input.fingerprint ?? `fingerprint-${input.deliveryId}` @@ -165,6 +167,12 @@ async function seedLegacyInboundEvent(input: { quotaDay: '2026-07-01', dedupeExpiresAt: '2026-09-01T00:00:00.000Z', state: input.state, + ...(input.cleanupRetryAt == null + ? {} + : { cleanupRetryAt: input.cleanupRetryAt }), + ...(input.reconcileAfter == null + ? {} + : { reconcileAfter: input.reconcileAfter }), ...(input.state === 'storing' ? { storageLease: 'legacy-storage-lease', @@ -190,11 +198,12 @@ async function seedLegacyInboundEvent(input: { id, message_id, user_id, inbox_id, event_type, provider, provider_event_id, detail_json, needs_effect_reconcile, state, fingerprint, storage_lease, storage_lease_at, - expected_attachment_count, finalization_token, dedupe_expires_at, + cleanup_retry_at, expected_attachment_count, finalization_token, + reconcile_after, dedupe_expires_at, usage_effect_recorded_at, usage_started_at, usage_month, usage_bytes, usage_duration_ms, subscription_effect_state, created_at, updated_at ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )`, ) .bind( @@ -211,10 +220,12 @@ async function seedLegacyInboundEvent(input: { fingerprint, 'storageLease' in detail ? detail.storageLease : null, 'storageLeaseAt' in detail ? detail.storageLeaseAt : null, + 'cleanupRetryAt' in detail ? detail.cleanupRetryAt : null, 'expectedAttachmentCount' in detail ? detail.expectedAttachmentCount : null, 'finalizationToken' in detail ? detail.finalizationToken : null, + 'reconcileAfter' in detail ? detail.reconcileAfter : null, detail.dedupeExpiresAt, 'usageEffectRecordedAt' in detail ? detail.usageEffectRecordedAt : null, 'usageStartedAt' in detail ? detail.usageStartedAt : null, @@ -647,17 +658,22 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi const pendingId = 'email-inbound-delivery:legacy-pending' const storingId = 'email-inbound-delivery:legacy-storing' const receivedId = 'email-inbound-delivery:legacy-received' + const orphanId = 'email-inbound-delivery:legacy-orphan-cleaned' + const reconcileAfter = '2026-08-03T12:00:00.000Z' + const cleanupRetryAt = '2026-08-03T13:00:00.000Z' const pending = await seedLegacyInboundEvent({ userId, deliveryId: pendingId, state: 'pending', createdAt: '2026-07-01T12:00:00.000Z', + reconcileAfter, }) await seedLegacyInboundEvent({ userId, deliveryId: storingId, state: 'storing', createdAt: '2026-07-01T12:01:00.000Z', + reconcileAfter, }) await seedLegacyInboundEvent({ userId, @@ -666,6 +682,14 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi state: 'received', createdAt: '2026-07-01T12:03:00.000Z', }) + await seedLegacyInboundEvent({ + userId, + deliveryId: orphanId, + state: 'orphan-cleaned', + createdAt: '2026-07-01T12:04:00.000Z', + cleanupRetryAt, + reconcileAfter, + }) await seedLegacyInboundEvent({ userId, deliveryId: pendingId, @@ -681,18 +705,11 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi }) const mailbox = rpcFor(userId) - expect(await mailbox.countMailbox()).toEqual({ - threads: 0, - messages: 0, - attachments: 0, - deliveryEvents: 0, - }) + expect((await mailbox.countMailbox()).deliveryEvents).toBe(0) const firstNow = new Date('2026-08-02T15:00:00.000Z') await expect( reconcileMailboxParity({ env, now: firstNow, batchSize: 1 }), ).resolves.toMatchObject({ - scanned: 1, - compared: 1, matched: 1, mismatched: 0, failed: 0, @@ -705,7 +722,7 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi ownerId: userId, deliveryId: pendingId, }), - ).toMatchObject({ state: 'pending' }) + ).toMatchObject({ state: 'pending', reconcileAfter }) expect( await mailbox.getInboundDelivery({ ownerId: userId, @@ -715,6 +732,7 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi state: 'storing', storageLease: 'legacy-storage-lease', expectedAttachmentCount: 2, + reconcileAfter, }) expect( await mailbox.getInboundDelivery({ @@ -727,6 +745,25 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi usageEffectRecordedAt: '2026-07-01T12:04:00.000Z', subscriptionEffectState: 'complete', }) + expect( + await mailbox.getInboundDelivery({ + ownerId: userId, + deliveryId: orphanId, + }), + ).toMatchObject({ + state: 'orphan-cleaned', + cleanupRetryAt, + reconcileAfter, + }) + const unboundEvents = await mailbox.listDeliveryEvents({ + messageId: null, + limit: 20, + }) + expect( + JSON.parse( + unboundEvents.find((event) => event.id === orphanId)!.detailJson, + ), + ).toMatchObject({ cleanupRetryAt, reconcileAfter }) expect( await mailbox.getInboundDeliveryWindow({ ownerId: userId, @@ -734,11 +771,22 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi now: '2026-08-02T15:00:00.000Z', }), ).toMatchObject({ deliveryId: pendingId }) + expect(unboundEvents.some((event) => event.id === auditEventId)).toBe(true) + await expect( + mailbox.listDueStaleInboundDeliveries({ + ownerId: userId, + now: '2026-08-02T16:00:00.000Z', + limit: 20, + }), + ).resolves.toEqual({ deliveries: [] }) + const dueAfterSchedules = await mailbox.listDueStaleInboundDeliveries({ + ownerId: userId, + now: '2026-08-04T16:00:00.000Z', + limit: 20, + }) expect( - (await mailbox.listDeliveryEvents({ messageId: null, limit: 20 })).some( - (event) => event.id === auditEventId, - ), - ).toBe(true) + new Set(dueAfterSchedules.deliveries.map((row) => row.deliveryId)), + ).toEqual(new Set([pendingId, storingId, orphanId])) const replayEventBackfill = () => env.APP_DB.prepare( @@ -757,13 +805,17 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi await expect( reconcileMailboxParity({ env, now: secondNow, batchSize: 1 }), ).resolves.toMatchObject({ - scanned: 1, - compared: 1, matched: 1, mismatched: 0, failed: 0, }) expect(await mailbox.countMailbox()).toEqual(countAfterFirst) + expect( + await mailbox.getInboundDelivery({ + ownerId: userId, + deliveryId: orphanId, + }), + ).toMatchObject({ cleanupRetryAt, reconcileAfter }) await mailbox.claimInboundDeliveryStorage({ ownerId: userId, @@ -785,8 +837,6 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi await expect( reconcileMailboxParity({ env, now: thirdNow, batchSize: 1 }), ).resolves.toMatchObject({ - scanned: 1, - compared: 1, matched: 1, mismatched: 0, failed: 0, @@ -804,4 +854,43 @@ test('reconcileMailboxParity bootstraps production legacy USER inbound shapes wi expectedAttachmentCount: 7, updatedAt: newerAuthority?.updatedAt, }) + + const malformedUserId = await seedParityUser({ + email: `legacy-malformed-${crypto.randomUUID()}@example.test`, + checkedAt: null, + }) + await seedLegacyInboundEvent({ + userId: malformedUserId, + deliveryId: 'email-inbound-delivery:malformed-schedule', + state: 'pending', + createdAt: '2026-07-01T13:00:00.000Z', + reconcileAfter: 'not-a-timestamp', + }) + await seedPreClaimAuditEvent({ + userId: malformedUserId, + inboxId: 'malformed-schedule-inbox', + createdAt: '2026-07-01T13:01:00.000Z', + }) + const malformedMailbox = rpcFor(malformedUserId) + await expect( + reconcileMailboxParity({ + env, + now: new Date('2026-08-02T17:00:00.000Z'), + batchSize: 1, + }), + ).resolves.toMatchObject({ + compared: 1, + matched: 0, + mismatched: 1, + failed: 0, + }) + expect( + (await countD1MailboxParity({ db: env.APP_DB, userId: malformedUserId })) + .deliveryEvents, + ).toBe(2) + expect((await malformedMailbox.countMailbox()).deliveryEvents).toBe(0) + expect(await readParityState(malformedUserId)).toMatchObject({ + mismatchCount: 1, + lastError: null, + }) }, 30_000)